{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5ec6afa8-5ad9-49be-a19e-b5c15925da49",
   "metadata": {},
   "source": [
    "# Benchmark the accuracy and speed of the package\n",
    "\n",
    "We measure the accuracy and the speed of the NodeGAM, Spline, EBM, and XGB-GAM. Everything is run on a machine with 12-core Intel(R) Xeon(R) W-2133 CPU @ 3.60GHz, 16GB RAM, and a Titan XP GPU with cuda 11.2. I use the default hyperparameter, so the performance is lower than what paper reported.\n",
    "\n",
    "Compare the models on the following datasets:\n",
    "- 3 classification and 3 regression datasets\n",
    "\n",
    "|           |   N  |  P |  Domain |     Problem    |\n",
    "|:---------:|:----:|:--:|:-------:|:--------------:|\n",
    "|   Mimic3  |  27K | 57 |  Health | Classification |\n",
    "|   Adult   |  33K | 14 | Finance | Classification |\n",
    "|   Credit  | 285K | 30 |  Retail | Classification |\n",
    "|    Wine   |  5K  | 16 |  Nature |   Regression   |\n",
    "| Bikeshare |  17K | 12 |  Retail |   Regression   |\n",
    "|    Year   | 515K | 90 |  Music  |   Regression   |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "40d3b772-a8ce-4cd9-b869-7be118513876",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "df76ac3a-ba35-4ff0-bb89-4de11a6b48a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from nodegam.data import DATASETS\n",
    "from nodegam.sklearn import NodeGAMRegressor, NodeGAMClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4aef2b51-b2b6-4b84-b67f-ec049606c2de",
   "metadata": {},
   "source": [
    "# Quick benchmark to find a good default hyperparameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc407746-6910-401b-b214-60a60d948494",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nodegam.gams.MyEBM import MyExplainableBoostingRegressor\n",
    "from nodegam.gams.MyXGB import MyXGBOnehotClassifier, MyXGBOnehotRegressor\n",
    "from nodegam.gams.MySpline import MySplineGAM, MySplineLogisticGAM\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b21c1973-399b-434c-a883-6428b1d7f763",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(data_name, model_name, fold=0, seed=31):\n",
    "    dataset = DATASETS[data_name.upper()](path='./data/', fold=fold)\n",
    "    \n",
    "    st_time = time.time()\n",
    "    \n",
    "    if model_name == 'nodegam':\n",
    "        model_cls = NodeGAMClassifier if dataset['problem'] == 'classification' \\\n",
    "            else NodeGAMRegressor\n",
    "    \n",
    "        model = model_cls(\n",
    "            arch='GAMAtt',\n",
    "            in_features=dataset['X_train'].shape[1],\n",
    "            cat_features=dataset.get('cat_features', None),\n",
    "            objective='negative_auc' if dataset['problem'] == 'classification' else 'mse',\n",
    "            ga2m=1,\n",
    "        )\n",
    "    elif model_name == 'ebm':\n",
    "        model_cls = MyExplainableBoostingClassifier if dataset['problem'] == 'classification' \\\n",
    "            else MyExplainableBoostingRegressor\n",
    "        model = model_cls()\n",
    "    elif model_name == 'xgb-gam':\n",
    "        model_cls = MyXGBOnehotClassifier if dataset['problem'] == 'classification' \\\n",
    "            else MyXGBOnehotRegressor\n",
    "        model = model_cls()\n",
    "    elif model_name == 'xgb':\n",
    "        model_cls = MyXGBOnehotClassifier if dataset['problem'] == 'classification' \\\n",
    "            else MyXGBOnehotRegressor\n",
    "        model = model_cls(max_depth=3)\n",
    "    elif model_name == 'spline':\n",
    "        model_cls = MySplineLogisticGAM if dataset['problem'] == 'classification' \\\n",
    "            else MySplineGAM\n",
    "        model = model_cls()\n",
    "    else:\n",
    "        raise NotImplementedError()\n",
    "        \n",
    "    model.fit(dataset['X_train'], dataset['y_train'])\n",
    "    \n",
    "    if dataset['problem'] == 'classification':\n",
    "        logit = model.predict_proba(dataset['X_test'])\n",
    "        if logit.ndim == 2:\n",
    "            logit = logit[:, 1]\n",
    "        test_perf = roc_auc_score(dataset['y_test'], logit)\n",
    "    else:\n",
    "        logit = model.predict(dataset['X_test'])\n",
    "        test_perf = np.sqrt(np.mean((logit - dataset['y_test']) ** 2))\n",
    "    \n",
    "    record = {}\n",
    "    record['dataset'] = data_name\n",
    "    record['model_name'] = model_name\n",
    "    record['fold'] = fold\n",
    "    record['seed'] = seed\n",
    "    record['test_perf'] = test_perf\n",
    "    record['time'] = round(float(time.time() - st_time), 0)\n",
    "    return record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "24e60106-a174-4bb5-884c-00af1c9e7d31",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dset = 'click'\n",
    "# model_name = 'nodegam'\n",
    "# fold=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39e965c6-d6eb-4db0-9bdd-9dbbb1047ea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# record = run(data_name=dset, model_name=model_name, fold=fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b833f44e-70c0-48ba-a4a7-b701da6db8c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "records = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ea9640f2-b401-4d8b-b047-b377ba063e17",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/qhoptim/pyt/qhadam.py:133: UserWarning: This overload of add_ is deprecated:\n",
      "\tadd_(Number alpha, Tensor other)\n",
      "Consider using one of the following signatures instead:\n",
      "\tadd_(Tensor other, *, Number alpha) (Triggered internally at  /opt/conda/conda-bld/pytorch_1607370116979/work/torch/csrc/utils/python_arg_parser.cpp:882.)\n",
      "  exp_avg.mul_(beta1_adj).add_(1.0 - beta1_adj, d_p)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.3948\t-0.8618\n",
      "200\t0.3541\t-0.8812\n",
      "300\t0.3191\t-0.8865\n",
      "400\t0.3187\t-0.8888\n",
      "500\t0.331\t-0.8894\n",
      "600\t0.3059\t-0.8957\n",
      "700\t0.3349\t-0.8972\n",
      "800\t0.3126\t-0.9038\n",
      "900\t0.3029\t-0.9076\n",
      "1000\t0.2999\t-0.9093\n",
      "1100\t0.2955\t-0.9104\n",
      "1200\t0.312\t-0.9112\n",
      "1300\t0.3221\t-0.9118\n",
      "1400\t0.33\t-0.9125\n",
      "1500\t0.3128\t-0.9131\n",
      "1600\t0.3261\t-0.9138\n",
      "1700\t0.3184\t-0.9144\n",
      "1800\t0.3025\t-0.9147\n",
      "1900\t0.2968\t-0.9147\n",
      "2000\t0.2951\t-0.9146\n",
      "2100\t0.3173\t-0.9144\n",
      "2200\t0.2939\t-0.9145\n",
      "2300\t0.3111\t-0.9148\n",
      "2400\t0.3072\t-0.9149\n",
      "2500\t0.3156\t-0.915\n",
      "2600\t0.3165\t-0.9151\n",
      "2700\t0.3154\t-0.9152\n",
      "2800\t0.292\t-0.9152\n",
      "2900\t0.2965\t-0.9153\n",
      "3000\t0.3038\t-0.9153\n",
      "3100\t0.3173\t-0.9155\n",
      "3200\t0.3033\t-0.9156\n",
      "3300\t0.3101\t-0.9158\n",
      "3400\t0.3003\t-0.9159\n",
      "3500\t0.286\t-0.916\n",
      "3600\t0.2891\t-0.9161\n",
      "3700\t0.2896\t-0.9162\n",
      "3800\t0.319\t-0.9169\n",
      "3900\t0.2993\t-0.9172\n",
      "4000\t0.2685\t-0.9173\n",
      "4100\t0.2981\t-0.9174\n",
      "4200\t0.2885\t-0.9176\n",
      "4300\t0.3014\t-0.9174\n",
      "4400\t0.3028\t-0.9176\n",
      "4500\t0.2982\t-0.9176\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "4600\t0.2744\t-0.9177\n",
      "4700\t0.3276\t-0.9176\n",
      "4800\t0.2795\t-0.9176\n",
      "4900\t0.2924\t-0.9175\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "5000\t0.3037\t-0.9176\n",
      "5100\t0.3014\t-0.9177\n",
      "5200\t0.3144\t-0.9177\n",
      "5300\t0.2941\t-0.9177\n",
      "5400\t0.2918\t-0.9177\n",
      "5500\t0.279\t-0.9177\n",
      "5600\t0.2793\t-0.9178\n",
      "5700\t0.2945\t-0.9178\n",
      "5800\t0.2981\t-0.9177\n",
      "5900\t0.2907\t-0.9177\n",
      "6000\t0.2963\t-0.9177\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "6100\t0.2903\t-0.9177\n",
      "6200\t0.2808\t-0.9176\n",
      "6300\t0.2874\t-0.9176\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "6400\t0.3008\t-0.9176\n",
      "6500\t0.2749\t-0.9176\n",
      "6600\t0.3015\t-0.9176\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "6700\t0.2882\t-0.9176\n",
      "6800\t0.2754\t-0.9176\n",
      "6900\t0.3012\t-0.9176\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "7000\t0.2927\t-0.9176\n",
      "7100\t0.283\t-0.9176\n",
      "7200\t0.298\t-0.9176\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "7300\t0.2769\t-0.9176\n",
      "7400\t0.2737\t-0.9176\n",
      "7500\t0.2977\t-0.9176\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "7600\t0.2818\t-0.9176\n",
      "7700\t0.3066\t-0.9176\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 157.6 seconds\n",
      "Best step:  5700\n",
      "Best Val Metric:  -0.9177744086196997\n",
      "Load the best checkpoint.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.3938\t-0.8722\n",
      "200\t0.3545\t-0.8936\n",
      "300\t0.3485\t-0.8974\n",
      "400\t0.312\t-0.9004\n",
      "500\t0.3163\t-0.9005\n",
      "600\t0.3148\t-0.9048\n",
      "700\t0.3024\t-0.9087\n",
      "800\t0.3263\t-0.9122\n",
      "900\t0.3352\t-0.9155\n",
      "1000\t0.315\t-0.9149\n",
      "1100\t0.3036\t-0.9158\n",
      "1200\t0.3139\t-0.9153\n",
      "1300\t0.3126\t-0.9153\n",
      "1400\t0.3049\t-0.9182\n",
      "1500\t0.302\t-0.9196\n",
      "1600\t0.3129\t-0.9207\n",
      "1700\t0.3065\t-0.9213\n",
      "1800\t0.3067\t-0.9217\n",
      "1900\t0.2973\t-0.9218\n",
      "2000\t0.3016\t-0.9218\n",
      "2100\t0.3087\t-0.9218\n",
      "2200\t0.2926\t-0.9219\n",
      "2300\t0.3019\t-0.9219\n",
      "2400\t0.3009\t-0.9221\n",
      "2500\t0.2807\t-0.9222\n",
      "2600\t0.2933\t-0.9222\n",
      "2700\t0.2971\t-0.9222\n",
      "2800\t0.2893\t-0.9223\n",
      "2900\t0.3016\t-0.9225\n",
      "3000\t0.2995\t-0.923\n",
      "3100\t0.3025\t-0.9233\n",
      "3200\t0.3112\t-0.9233\n",
      "3300\t0.2758\t-0.9234\n",
      "3400\t0.2897\t-0.9233\n",
      "3500\t0.3241\t-0.9234\n",
      "3600\t0.2897\t-0.9233\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "3700\t0.2802\t-0.9234\n",
      "3800\t0.2831\t-0.9234\n",
      "3900\t0.2914\t-0.9235\n",
      "4000\t0.2972\t-0.9235\n",
      "4100\t0.3121\t-0.9234\n",
      "4200\t0.2966\t-0.9235\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "4300\t0.3009\t-0.9235\n",
      "4400\t0.2987\t-0.9235\n",
      "4500\t0.2818\t-0.9235\n",
      "4600\t0.2901\t-0.9235\n",
      "4700\t0.3032\t-0.9235\n",
      "4800\t0.3023\t-0.9235\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "4900\t0.296\t-0.9235\n",
      "5000\t0.2896\t-0.9235\n",
      "5100\t0.3068\t-0.9235\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "5200\t0.2955\t-0.9235\n",
      "5300\t0.3234\t-0.9235\n",
      "5400\t0.2981\t-0.9235\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "5500\t0.2901\t-0.9235\n",
      "5600\t0.2956\t-0.9235\n",
      "5700\t0.2922\t-0.9235\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "5800\t0.2885\t-0.9235\n",
      "5900\t0.2872\t-0.9235\n",
      "6000\t0.2963\t-0.9235\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "6100\t0.2763\t-0.9235\n",
      "6200\t0.3086\t-0.9235\n",
      "6300\t0.2875\t-0.9235\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "6400\t0.2769\t-0.9235\n",
      "6500\t0.2825\t-0.9235\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 136.5 seconds\n",
      "Best step:  4500\n",
      "Best Val Metric:  -0.9235438574174581\n",
      "Load the best checkpoint.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.3825\t-0.8695\n",
      "200\t0.375\t-0.8894\n",
      "300\t0.3572\t-0.8957\n",
      "400\t0.3273\t-0.8992\n",
      "500\t0.324\t-0.9016\n",
      "600\t0.3084\t-0.9072\n",
      "700\t0.3072\t-0.9138\n",
      "800\t0.3043\t-0.9187\n",
      "900\t0.3191\t-0.9196\n",
      "1000\t0.3119\t-0.919\n",
      "1100\t0.2986\t-0.9202\n",
      "1200\t0.3126\t-0.9209\n",
      "1300\t0.3246\t-0.9224\n",
      "1400\t0.3059\t-0.9237\n",
      "1500\t0.3021\t-0.9246\n",
      "1600\t0.3136\t-0.9245\n",
      "1700\t0.3021\t-0.9248\n",
      "1800\t0.3073\t-0.9252\n",
      "1900\t0.3118\t-0.9254\n",
      "2000\t0.3136\t-0.9254\n",
      "2100\t0.2846\t-0.9255\n",
      "2200\t0.2857\t-0.9255\n",
      "2300\t0.3016\t-0.9255\n",
      "2400\t0.2933\t-0.9257\n",
      "2500\t0.2972\t-0.9258\n",
      "2600\t0.298\t-0.9258\n",
      "2700\t0.3155\t-0.9258\n",
      "2800\t0.2802\t-0.9257\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "2900\t0.3026\t-0.9257\n",
      "3000\t0.3105\t-0.9258\n",
      "3100\t0.2993\t-0.9258\n",
      "3200\t0.2816\t-0.926\n",
      "3300\t0.3081\t-0.9261\n",
      "3400\t0.312\t-0.9262\n",
      "3500\t0.3021\t-0.9261\n",
      "3600\t0.2849\t-0.9262\n",
      "3700\t0.2972\t-0.9262\n",
      "3800\t0.2989\t-0.9262\n",
      "3900\t0.2843\t-0.9263\n",
      "4000\t0.2905\t-0.9264\n",
      "4100\t0.2995\t-0.9264\n",
      "4200\t0.2888\t-0.9265\n",
      "4300\t0.2952\t-0.9266\n",
      "4400\t0.2941\t-0.9267\n",
      "4500\t0.2853\t-0.9267\n",
      "4600\t0.2786\t-0.9269\n",
      "4700\t0.3064\t-0.927\n",
      "4800\t0.31\t-0.927\n",
      "4900\t0.2968\t-0.927\n",
      "5000\t0.3029\t-0.9271\n",
      "5100\t0.2836\t-0.9271\n",
      "5200\t0.2985\t-0.9272\n",
      "5300\t0.2775\t-0.9272\n",
      "5400\t0.2898\t-0.9272\n",
      "5500\t0.2871\t-0.9273\n",
      "5600\t0.2943\t-0.9274\n",
      "5700\t0.2781\t-0.9274\n",
      "5800\t0.28\t-0.9274\n",
      "5900\t0.3091\t-0.9275\n",
      "6000\t0.2982\t-0.9275\n",
      "6100\t0.3042\t-0.9275\n",
      "6200\t0.2838\t-0.9276\n",
      "6300\t0.2903\t-0.9276\n",
      "6400\t0.2893\t-0.9276\n",
      "6500\t0.3198\t-0.9277\n",
      "6600\t0.2983\t-0.9277\n",
      "6700\t0.3153\t-0.9278\n",
      "6800\t0.283\t-0.9278\n",
      "6900\t0.2718\t-0.9279\n",
      "7000\t0.2956\t-0.9279\n",
      "7100\t0.2788\t-0.9279\n",
      "7200\t0.2885\t-0.928\n",
      "7300\t0.2964\t-0.928\n",
      "7400\t0.297\t-0.9281\n",
      "7500\t0.289\t-0.9282\n",
      "7600\t0.2934\t-0.9282\n",
      "7700\t0.2902\t-0.9282\n",
      "7800\t0.2826\t-0.9282\n",
      "7900\t0.2927\t-0.9281\n",
      "8000\t0.2799\t-0.928\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "8100\t0.3012\t-0.928\n",
      "8200\t0.2985\t-0.928\n",
      "8300\t0.3111\t-0.928\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "8400\t0.2971\t-0.928\n",
      "8500\t0.2918\t-0.9281\n",
      "8600\t0.2811\t-0.9281\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "8700\t0.279\t-0.9281\n",
      "8800\t0.3036\t-0.9282\n",
      "8900\t0.2856\t-0.9282\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "9000\t0.2831\t-0.9282\n",
      "9100\t0.292\t-0.9282\n",
      "9200\t0.2657\t-0.9282\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "9300\t0.3132\t-0.9282\n",
      "9400\t0.2961\t-0.9282\n",
      "9500\t0.2865\t-0.9282\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "9600\t0.2738\t-0.9282\n",
      "9700\t0.2949\t-0.9282\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 204.7 seconds\n",
      "Best step:  7700\n",
      "Best Val Metric:  -0.9281927861119615\n",
      "Load the best checkpoint.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/pandas/core/indexing.py:670: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  iloc._setitem_with_indexer(indexer, value)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.3136\t-0.8007\n",
      "200\t0.3063\t-0.8186\n",
      "300\t0.2822\t-0.8227\n",
      "400\t0.2895\t-0.8252\n",
      "500\t0.2782\t-0.8263\n",
      "600\t0.2732\t-0.8301\n",
      "700\t0.2688\t-0.8327\n",
      "800\t0.2629\t-0.8349\n",
      "900\t0.2235\t-0.836\n",
      "1000\t0.2936\t-0.8363\n",
      "1100\t0.2932\t-0.8379\n",
      "1200\t0.2799\t-0.8397\n",
      "1300\t0.2906\t-0.8406\n",
      "1400\t0.2791\t-0.8434\n",
      "1500\t0.2822\t-0.8446\n",
      "1600\t0.2883\t-0.8436\n",
      "1700\t0.2633\t-0.8437\n",
      "1800\t0.212\t-0.8431\n",
      "1900\t0.2865\t-0.8437\n",
      "2000\t0.282\t-0.8442\n",
      "2100\t0.2913\t-0.8447\n",
      "2200\t0.2972\t-0.845\n",
      "2300\t0.2604\t-0.845\n",
      "2400\t0.2759\t-0.845\n",
      "2500\t0.2869\t-0.845\n",
      "2600\t0.2864\t-0.8455\n",
      "2700\t0.2926\t-0.8449\n",
      "2800\t0.267\t-0.8447\n",
      "2900\t0.2687\t-0.8442\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "3000\t0.2706\t-0.8438\n",
      "3100\t0.2569\t-0.8434\n",
      "3200\t0.2514\t-0.8439\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "3300\t0.2691\t-0.8446\n",
      "3400\t0.2544\t-0.8452\n",
      "3500\t0.2519\t-0.8455\n",
      "3600\t0.2493\t-0.8455\n",
      "3700\t0.2687\t-0.8454\n",
      "3800\t0.2762\t-0.8453\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "3900\t0.2492\t-0.8451\n",
      "4000\t0.2525\t-0.845\n",
      "4100\t0.2769\t-0.8449\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "4200\t0.2645\t-0.8448\n",
      "4300\t0.2721\t-0.8449\n",
      "4400\t0.2613\t-0.8448\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "4500\t0.27\t-0.8449\n",
      "4600\t0.2663\t-0.8448\n",
      "4700\t0.2662\t-0.8448\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "4800\t0.276\t-0.8448\n",
      "4900\t0.2609\t-0.8448\n",
      "5000\t0.2638\t-0.8448\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "5100\t0.2601\t-0.8448\n",
      "5200\t0.2502\t-0.8448\n",
      "5300\t0.2769\t-0.8448\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "5400\t0.2116\t-0.8448\n",
      "5500\t0.2497\t-0.8448\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 112.2 seconds\n",
      "Best step:  3500\n",
      "Best Val Metric:  -0.8455344776499936\n",
      "Load the best checkpoint.\n",
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.3275\t-0.8106\n",
      "200\t0.278\t-0.8212\n",
      "300\t0.3114\t-0.8262\n",
      "400\t0.3074\t-0.8313\n",
      "500\t0.3068\t-0.8329\n",
      "600\t0.2856\t-0.8366\n",
      "700\t0.2734\t-0.8402\n",
      "800\t0.2651\t-0.8379\n",
      "900\t0.2677\t-0.8367\n",
      "1000\t0.2667\t-0.8353\n",
      "1100\t0.2883\t-0.8366\n",
      "1200\t0.2606\t-0.8382\n",
      "1300\t0.267\t-0.8398\n",
      "1400\t0.2553\t-0.8419\n",
      "1500\t0.2592\t-0.8422\n",
      "1600\t0.2623\t-0.8439\n",
      "1700\t0.2665\t-0.8415\n",
      "1800\t0.258\t-0.843\n",
      "1900\t0.2708\t-0.8445\n",
      "2000\t0.2729\t-0.8448\n",
      "2100\t0.2604\t-0.8458\n",
      "2200\t0.2539\t-0.8464\n",
      "2300\t0.2515\t-0.8465\n",
      "2400\t0.2398\t-0.8458\n",
      "2500\t0.2562\t-0.8458\n",
      "2600\t0.2497\t-0.845\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "2700\t0.2896\t-0.8447\n",
      "2800\t0.2433\t-0.8449\n",
      "2900\t0.2629\t-0.8456\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "3000\t0.2329\t-0.8456\n",
      "3100\t0.2595\t-0.8457\n",
      "3200\t0.2662\t-0.8458\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "3300\t0.2605\t-0.8458\n",
      "3400\t0.2486\t-0.8457\n",
      "3500\t0.241\t-0.8456\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "3600\t0.3038\t-0.8456\n",
      "3700\t0.2412\t-0.8456\n",
      "3800\t0.2539\t-0.8456\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "3900\t0.2544\t-0.8456\n",
      "4000\t0.2475\t-0.8456\n",
      "4100\t0.2483\t-0.8456\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "4200\t0.2319\t-0.8456\n",
      "4300\t0.2494\t-0.8456\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 88.5 seconds\n",
      "Best step:  2300\n",
      "Best Val Metric:  -0.8464563146021286\n",
      "Load the best checkpoint.\n",
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.3001\t-0.7995\n",
      "200\t0.3229\t-0.8116\n",
      "300\t0.3195\t-0.8154\n",
      "400\t0.2692\t-0.8207\n",
      "500\t0.281\t-0.8229\n",
      "600\t0.2511\t-0.8262\n",
      "700\t0.2805\t-0.8271\n",
      "800\t0.2742\t-0.8325\n",
      "900\t0.2877\t-0.8332\n",
      "1000\t0.2654\t-0.8366\n",
      "1100\t0.2686\t-0.8356\n",
      "1200\t0.2747\t-0.8347\n",
      "1300\t0.275\t-0.8354\n",
      "1400\t0.2571\t-0.8381\n",
      "1500\t0.2782\t-0.8364\n",
      "1600\t0.2727\t-0.8379\n",
      "1700\t0.2758\t-0.8393\n",
      "1800\t0.2385\t-0.8407\n",
      "1900\t0.2929\t-0.8413\n",
      "2000\t0.2709\t-0.842\n",
      "2100\t0.2667\t-0.8422\n",
      "2200\t0.2451\t-0.8426\n",
      "2300\t0.2532\t-0.8426\n",
      "2400\t0.2764\t-0.8423\n",
      "2500\t0.2657\t-0.8423\n",
      "2600\t0.2698\t-0.8428\n",
      "2700\t0.2125\t-0.8437\n",
      "2800\t0.2512\t-0.844\n",
      "2900\t0.2604\t-0.8441\n",
      "3000\t0.2689\t-0.8438\n",
      "3100\t0.2606\t-0.8443\n",
      "3200\t0.2544\t-0.8444\n",
      "3300\t0.2677\t-0.845\n",
      "3400\t0.2779\t-0.8455\n",
      "3500\t0.2651\t-0.8461\n",
      "3600\t0.3431\t-0.8456\n",
      "3700\t0.2486\t-0.8456\n",
      "3800\t0.2791\t-0.8453\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "3900\t0.2669\t-0.8449\n",
      "4000\t0.2416\t-0.8451\n",
      "4100\t0.2496\t-0.8458\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "4200\t0.2672\t-0.8453\n",
      "4300\t0.2498\t-0.845\n",
      "4400\t0.2409\t-0.8453\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "4500\t0.2524\t-0.8453\n",
      "4600\t0.2811\t-0.8452\n",
      "4700\t0.2643\t-0.8451\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "4800\t0.2355\t-0.8451\n",
      "4900\t0.2466\t-0.8451\n",
      "5000\t0.2398\t-0.8451\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "5100\t0.287\t-0.8451\n",
      "5200\t0.2559\t-0.8451\n",
      "5300\t0.2536\t-0.8451\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "5400\t0.2095\t-0.8451\n",
      "5500\t0.249\t-0.8451\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 113.5 seconds\n",
      "Best step:  3500\n",
      "Best Val Metric:  -0.8460789366316648\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 5.878509521484375, std = 0.8946797847747803\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.8971\t0.71\n",
      "200\t1.482\t0.6604\n",
      "300\t0.6245\t0.6288\n",
      "400\t0.6667\t0.5975\n",
      "500\t0.603\t0.5974\n",
      "600\t0.5803\t0.5843\n",
      "700\t0.5808\t0.5787\n",
      "800\t0.5764\t0.5766\n",
      "900\t0.6259\t0.5654\n",
      "1000\t0.5786\t0.5663\n",
      "1100\t0.5884\t0.5611\n",
      "1200\t0.5877\t0.5602\n",
      "1300\t0.6075\t0.5572\n",
      "1400\t0.5303\t0.5446\n",
      "1500\t0.6067\t0.5424\n",
      "1600\t0.559\t0.5366\n",
      "1700\t0.5109\t0.5345\n",
      "1800\t0.5434\t0.5264\n",
      "1900\t0.5195\t0.5258\n",
      "2000\t0.5487\t0.5249\n",
      "2100\t0.5667\t0.5244\n",
      "2200\t0.5853\t0.5235\n",
      "2300\t0.5205\t0.5234\n",
      "2400\t0.5052\t0.5235\n",
      "2500\t0.548\t0.5252\n",
      "2600\t0.5323\t0.5274\n",
      "2700\t0.5375\t0.5288\n",
      "2800\t0.5336\t0.5314\n",
      "2900\t0.5373\t0.5305\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "3000\t0.515\t0.5267\n",
      "3100\t0.5208\t0.5263\n",
      "3200\t0.4984\t0.5247\n",
      "3300\t0.5155\t0.5232\n",
      "3400\t0.5052\t0.5235\n",
      "3500\t0.5043\t0.5234\n",
      "3600\t0.5415\t0.5225\n",
      "3700\t0.5216\t0.5217\n",
      "3800\t0.5503\t0.521\n",
      "3900\t0.5602\t0.5194\n",
      "4000\t0.5071\t0.5179\n",
      "4100\t0.5524\t0.5171\n",
      "4200\t0.5421\t0.5165\n",
      "4300\t0.4947\t0.515\n",
      "4400\t0.5056\t0.5141\n",
      "4500\t0.5232\t0.5143\n",
      "4600\t0.5285\t0.5139\n",
      "4700\t0.4913\t0.5134\n",
      "4800\t0.488\t0.5138\n",
      "4900\t0.5014\t0.5145\n",
      "5000\t0.493\t0.5135\n",
      "5100\t0.4781\t0.5133\n",
      "5200\t0.4905\t0.5125\n",
      "5300\t0.4575\t0.5125\n",
      "5400\t0.5421\t0.5123\n",
      "5500\t0.4904\t0.5123\n",
      "5600\t0.5069\t0.5125\n",
      "5700\t0.513\t0.5131\n",
      "5800\t0.475\t0.5119\n",
      "5900\t0.5157\t0.511\n",
      "6000\t0.5333\t0.5108\n",
      "6100\t0.4911\t0.5106\n",
      "6200\t0.5099\t0.5112\n",
      "6300\t0.5107\t0.511\n",
      "6400\t0.5019\t0.5115\n",
      "6500\t0.4925\t0.5113\n",
      "6600\t0.5065\t0.5106\n",
      "6700\t0.4787\t0.5097\n",
      "6800\t0.4905\t0.5091\n",
      "6900\t0.4963\t0.5089\n",
      "7000\t0.4745\t0.5078\n",
      "7100\t0.477\t0.5078\n",
      "7200\t0.4547\t0.5071\n",
      "7300\t0.5036\t0.5069\n",
      "7400\t0.4666\t0.5065\n",
      "7500\t0.5175\t0.5071\n",
      "7600\t0.5006\t0.5064\n",
      "7700\t0.4621\t0.5058\n",
      "7800\t0.501\t0.5058\n",
      "7900\t0.5341\t0.5061\n",
      "8000\t0.4778\t0.5055\n",
      "8100\t0.4731\t0.5053\n",
      "8200\t0.5296\t0.5059\n",
      "8300\t0.4983\t0.5061\n",
      "8400\t0.5036\t0.506\n",
      "8500\t0.527\t0.5069\n",
      "8600\t0.4669\t0.5084\n",
      "8700\t0.481\t0.5076\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "8800\t0.4679\t0.5074\n",
      "8900\t0.4269\t0.5066\n",
      "9000\t0.4715\t0.5052\n",
      "9100\t0.4768\t0.5045\n",
      "9200\t0.534\t0.5043\n",
      "9300\t0.4688\t0.5053\n",
      "9400\t0.4981\t0.5054\n",
      "9500\t0.4444\t0.5054\n",
      "9600\t0.4834\t0.5049\n",
      "9700\t0.454\t0.5052\n",
      "9800\t0.4464\t0.5055\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "9900\t0.4823\t0.5055\n",
      "10000\t0.4873\t0.5056\n",
      "10100\t0.4332\t0.5053\n",
      "10200\t0.5106\t0.5052\n",
      "10300\t0.4521\t0.5048\n",
      "10400\t0.4681\t0.5046\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "10500\t0.49\t0.5044\n",
      "10600\t0.4501\t0.5043\n",
      "10700\t0.4897\t0.5042\n",
      "10800\t0.542\t0.5041\n",
      "10900\t0.5041\t0.504\n",
      "11000\t0.4876\t0.504\n",
      "11100\t0.464\t0.5041\n",
      "11200\t0.4892\t0.5041\n",
      "11300\t0.4941\t0.5041\n",
      "11400\t0.4945\t0.5041\n",
      "11500\t0.459\t0.5042\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "11600\t0.4731\t0.5042\n",
      "11700\t0.4811\t0.5042\n",
      "11800\t0.4528\t0.5042\n",
      "11900\t0.4659\t0.5042\n",
      "12000\t0.486\t0.5042\n",
      "12100\t0.481\t0.5043\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "12200\t0.4858\t0.5043\n",
      "12300\t0.4691\t0.5043\n",
      "12400\t0.4935\t0.5043\n",
      "12500\t0.4663\t0.5043\n",
      "12600\t0.4649\t0.5043\n",
      "12700\t0.4875\t0.5043\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "12800\t0.4946\t0.5043\n",
      "12900\t0.4871\t0.5043\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 248.0 seconds\n",
      "Best step:  10900\n",
      "Best Val Metric:  0.504008173132385\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 5.888974189758301, std = 0.8821154236793518\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.8563\t0.7167\n",
      "200\t0.8957\t0.7494\n",
      "300\t0.6901\t0.6582\n",
      "400\t0.5901\t0.6751\n",
      "500\t0.6239\t0.6584\n",
      "600\t0.6646\t0.6184\n",
      "700\t0.5957\t0.5742\n",
      "800\t0.6316\t0.5641\n",
      "900\t0.5915\t0.5541\n",
      "1000\t0.6162\t0.5485\n",
      "1100\t0.5681\t0.5514\n",
      "1200\t0.5792\t0.548\n",
      "1300\t0.5831\t0.5513\n",
      "1400\t0.5771\t0.5501\n",
      "1500\t0.5614\t0.5516\n",
      "1600\t0.5708\t0.5566\n",
      "1700\t0.5536\t0.5459\n",
      "1800\t0.5636\t0.5369\n",
      "1900\t0.5667\t0.528\n",
      "2000\t0.5521\t0.5203\n",
      "2100\t0.5624\t0.5138\n",
      "2200\t0.5483\t0.5133\n",
      "2300\t0.5359\t0.5154\n",
      "2400\t0.5323\t0.5181\n",
      "2500\t0.543\t0.5187\n",
      "2600\t0.507\t0.5177\n",
      "2700\t0.5867\t0.5171\n",
      "2800\t0.5637\t0.5175\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "2900\t0.5061\t0.5176\n",
      "3000\t0.5086\t0.5171\n",
      "3100\t0.4924\t0.5156\n",
      "3200\t0.4994\t0.5138\n",
      "3300\t0.4931\t0.5137\n",
      "3400\t0.4883\t0.5138\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "3500\t0.5519\t0.5137\n",
      "3600\t0.5277\t0.5139\n",
      "3700\t0.521\t0.5141\n",
      "3800\t0.5083\t0.5142\n",
      "3900\t0.4841\t0.514\n",
      "4000\t0.5359\t0.5138\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "4100\t0.5209\t0.5136\n",
      "4200\t0.5035\t0.5133\n",
      "4300\t0.5309\t0.513\n",
      "4400\t0.5423\t0.5129\n",
      "4500\t0.5228\t0.5129\n",
      "4600\t0.536\t0.5129\n",
      "4700\t0.56\t0.5129\n",
      "4800\t0.5275\t0.5128\n",
      "4900\t0.5201\t0.5127\n",
      "5000\t0.5091\t0.5127\n",
      "5100\t0.5285\t0.5125\n",
      "5200\t0.4945\t0.5124\n",
      "5300\t0.4918\t0.5123\n",
      "5400\t0.5335\t0.5122\n",
      "5500\t0.4879\t0.5122\n",
      "5600\t0.4701\t0.5122\n",
      "5700\t0.5551\t0.5123\n",
      "5800\t0.5448\t0.5124\n",
      "5900\t0.5032\t0.5125\n",
      "6000\t0.4632\t0.5126\n",
      "6100\t0.553\t0.5125\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "6200\t0.485\t0.5125\n",
      "6300\t0.4994\t0.5125\n",
      "6400\t0.4982\t0.5124\n",
      "6500\t0.5248\t0.5124\n",
      "6600\t0.5149\t0.5123\n",
      "6700\t0.5525\t0.5123\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "6800\t0.4965\t0.5124\n",
      "6900\t0.5161\t0.5124\n",
      "7000\t0.5373\t0.5124\n",
      "7100\t0.5589\t0.5124\n",
      "7200\t0.5437\t0.5124\n",
      "7300\t0.5024\t0.5124\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "7400\t0.5127\t0.5124\n",
      "7500\t0.4836\t0.5124\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 144.3 seconds\n",
      "Best step:  5500\n",
      "Best Val Metric:  0.5121652895086221\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 5.881317138671875, std = 0.8814074993133545\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.7422\t0.6345\n",
      "200\t0.6905\t0.585\n",
      "300\t0.7138\t0.5823\n",
      "400\t0.6525\t0.5368\n",
      "500\t0.6526\t0.5336\n",
      "600\t0.6653\t0.497\n",
      "700\t0.712\t0.497\n",
      "800\t0.6389\t0.4956\n",
      "900\t0.5947\t0.4938\n",
      "1000\t0.6173\t0.4832\n",
      "1100\t0.573\t0.4799\n",
      "1200\t0.6158\t0.4773\n",
      "1300\t0.5865\t0.4777\n",
      "1400\t0.6038\t0.4768\n",
      "1500\t0.5817\t0.4741\n",
      "1600\t0.5962\t0.4721\n",
      "1700\t0.5785\t0.4745\n",
      "1800\t0.6071\t0.4727\n",
      "1900\t0.545\t0.4777\n",
      "2000\t0.5532\t0.4781\n",
      "2100\t0.544\t0.4778\n",
      "2200\t0.5557\t0.4776\n",
      "2300\t0.5908\t0.4784\n",
      "2400\t0.5488\t0.4793\n",
      "2500\t0.5572\t0.4769\n",
      "2600\t0.5357\t0.477\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "2700\t0.5662\t0.4768\n",
      "2800\t0.5411\t0.4756\n",
      "2900\t0.5715\t0.4763\n",
      "3000\t0.4889\t0.4763\n",
      "3100\t0.535\t0.4764\n",
      "3200\t0.5446\t0.4768\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "3300\t0.5421\t0.4765\n",
      "3400\t0.5048\t0.4764\n",
      "3500\t0.5123\t0.4762\n",
      "3600\t0.5202\t0.4762\n",
      "3700\t0.4767\t0.4759\n",
      "3800\t0.5226\t0.476\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "3900\t0.5436\t0.476\n",
      "4000\t0.4973\t0.4761\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 78.1 seconds\n",
      "Best step:  1600\n",
      "Best Val Metric:  0.4720990975791395\n",
      "Load the best checkpoint.\n",
      "2600\t0.2339\t3351.6247\n",
      "2700\t0.1526\t3313.8631\n",
      "2800\t0.1075\t3312.591\n",
      "2900\t0.1241\t3318.1492\n",
      "3000\t0.1214\t3266.1209\n",
      "3100\t0.2912\t3255.1962\n",
      "3200\t0.1289\t3251.8657\n",
      "3300\t0.1396\t3242.0004\n",
      "3400\t0.2036\t3176.6979\n",
      "3500\t0.1245\t3184.8731\n",
      "3600\t0.2113\t3234.5358\n",
      "3700\t0.1234\t3183.3973\n",
      "3800\t0.1183\t3139.7771\n",
      "3900\t0.2148\t3102.0434\n",
      "4000\t0.1223\t3076.683\n",
      "4100\t0.13\t3052.5263\n",
      "4200\t0.1062\t3057.8159\n",
      "4300\t0.121\t3088.7202\n",
      "4400\t0.1247\t3087.625\n",
      "4500\t0.1155\t3066.5201\n",
      "4600\t0.1315\t3006.1124\n",
      "4700\t0.1523\t3004.9659\n",
      "4800\t0.1383\t3009.4542\n",
      "4900\t0.1403\t3021.0364\n",
      "5000\t0.1956\t2935.7387\n",
      "5100\t0.1418\t2910.149\n",
      "5200\t0.1276\t2893.1308\n",
      "5300\t0.1245\t2888.5608\n",
      "5400\t0.1921\t2843.9244\n",
      "5500\t0.12\t2847.019\n",
      "5600\t0.1636\t2883.4426\n",
      "5700\t0.1015\t2840.2629\n",
      "5800\t0.1117\t2830.5706\n",
      "5900\t0.1337\t2842.7085\n",
      "6000\t0.1199\t2844.5341\n",
      "6100\t0.1612\t2831.7587\n",
      "6200\t0.1915\t2840.5902\n",
      "6300\t0.1028\t2823.7693\n",
      "6400\t0.118\t2843.6829\n",
      "6500\t0.1184\t2821.3017\n",
      "6600\t0.1479\t2858.5528\n",
      "6700\t0.1024\t2889.909\n",
      "6800\t0.097\t2835.3764\n",
      "6900\t0.1379\t2840.8882\n",
      "7000\t0.1228\t2846.4582\n",
      "7100\t0.1082\t2831.0224\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "7200\t0.1224\t2791.3088\n",
      "7300\t0.0896\t2819.1033\n",
      "7400\t0.1164\t2802.9157\n",
      "7500\t0.1052\t2793.8896\n",
      "7600\t0.1567\t2772.5205\n",
      "7700\t0.1076\t2777.8536\n",
      "7800\t0.1049\t2766.6386\n",
      "7900\t0.0944\t2773.7423\n",
      "8000\t0.0983\t2770.5186\n",
      "8100\t0.12\t2777.3781\n",
      "8200\t0.1111\t2787.7344\n",
      "8300\t0.1015\t2799.0729\n",
      "8400\t0.114\t2792.9069\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "8500\t0.1102\t2795.2498\n",
      "8600\t0.1037\t2795.4202\n",
      "8700\t0.1019\t2784.0426\n",
      "8800\t0.0941\t2781.9627\n",
      "8900\t0.0892\t2784.1531\n",
      "9000\t0.1026\t2783.7424\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "9100\t0.1139\t2788.2619\n",
      "9200\t0.1096\t2791.7953\n",
      "9300\t0.0934\t2790.8106\n",
      "9400\t0.1441\t2790.1386\n",
      "9500\t0.1116\t2786.9873\n",
      "9600\t0.1011\t2783.3642\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "9700\t0.1042\t2781.2284\n",
      "9800\t0.0958\t2782.3076\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 197.3 seconds\n",
      "Best step:  7800\n",
      "Best Val Metric:  2766.63857196394\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 189.45860290527344, std = 181.07901000976562\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.6391\t29599.3884\n",
      "200\t0.6583\t19523.4555\n",
      "300\t0.3927\t16322.4516\n",
      "400\t0.2768\t15220.5125\n",
      "500\t0.1928\t14646.7164\n",
      "600\t0.1687\t12126.7365\n",
      "700\t0.279\t10729.8902\n",
      "800\t0.2673\t11374.9344\n",
      "900\t0.1382\t13579.6255\n",
      "1000\t0.1497\t15668.3692\n",
      "1100\t0.1397\t13046.0493\n",
      "1200\t0.1469\t10887.5468\n",
      "1300\t0.1443\t9226.9427\n",
      "1400\t0.1179\t7847.4223\n",
      "1500\t0.1581\t7098.375\n",
      "1600\t0.1634\t6787.9658\n",
      "1700\t0.1112\t6370.1988\n",
      "1800\t0.1944\t5203.7554\n",
      "1900\t0.1895\t4713.8666\n",
      "2000\t0.1309\t4297.198\n",
      "2100\t0.1672\t4116.2716\n",
      "2200\t0.1561\t3917.864\n",
      "2300\t0.1442\t3898.5207\n",
      "2400\t0.1239\t3840.6992\n",
      "2500\t0.1276\t3740.8702\n",
      "2600\t0.1282\t3705.7707\n",
      "2700\t0.1392\t3680.8929\n",
      "2800\t0.1219\t3689.6534\n",
      "2900\t0.2178\t3670.7825\n",
      "3000\t0.1359\t3650.7429\n",
      "3100\t0.1348\t3630.376\n",
      "3200\t0.1284\t3601.2438\n",
      "3300\t0.2221\t3557.2094\n",
      "3400\t0.1272\t3571.3261\n",
      "3500\t0.1111\t3596.6614\n",
      "3600\t0.1342\t3548.8057\n",
      "3700\t0.1228\t3562.4665\n",
      "3800\t0.1899\t3592.4743\n",
      "3900\t0.1217\t3614.857\n",
      "4000\t0.2081\t3646.958\n",
      "4100\t0.1117\t3702.6693\n",
      "4200\t0.1521\t3707.6081\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "4300\t0.1635\t3678.9021\n",
      "4400\t0.1469\t3634.9337\n",
      "4500\t0.1264\t3566.238\n",
      "4600\t0.1146\t3528.9354\n",
      "4700\t0.188\t3496.4682\n",
      "4800\t0.1262\t3484.2424\n",
      "4900\t0.1147\t3488.6549\n",
      "5000\t0.117\t3494.0584\n",
      "5100\t0.1531\t3491.4297\n",
      "5200\t0.1239\t3472.1467\n",
      "5300\t0.1281\t3474.0411\n",
      "5400\t0.1179\t3452.4194\n",
      "5500\t0.1784\t3453.2007\n",
      "5600\t0.1291\t3438.2255\n",
      "5700\t0.1698\t3430.8595\n",
      "5800\t0.0986\t3438.8826\n",
      "5900\t0.1142\t3455.6962\n",
      "6000\t0.1146\t3441.9513\n",
      "6100\t0.1278\t3447.7918\n",
      "6200\t0.1093\t3456.3644\n",
      "6300\t0.1067\t3423.776\n",
      "6400\t0.1246\t3409.9439\n",
      "6500\t0.1025\t3398.985\n",
      "6600\t0.1138\t3386.1201\n",
      "6700\t0.1155\t3359.7059\n",
      "6800\t0.0991\t3372.2782\n",
      "6900\t0.1211\t3342.5416\n",
      "7000\t0.1111\t3354.375\n",
      "7100\t0.1254\t3346.9783\n",
      "7200\t0.094\t3354.0469\n",
      "7300\t0.1476\t3319.4829\n",
      "7400\t0.1707\t3317.7569\n",
      "7500\t0.1376\t3311.9919\n",
      "7600\t0.1038\t3317.0045\n",
      "7700\t0.1236\t3303.3585\n",
      "7800\t0.1459\t3296.4936\n",
      "7900\t0.1577\t3308.3298\n",
      "8000\t0.105\t3303.3197\n",
      "8100\t0.2077\t3297.3326\n",
      "8200\t0.1185\t3310.4457\n",
      "8300\t0.1161\t3332.8257\n",
      "8400\t0.1397\t3317.6201\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "8500\t0.1063\t3317.5815\n",
      "8600\t0.1211\t3302.5833\n",
      "8700\t0.1088\t3291.3271\n",
      "8800\t0.0942\t3288.5692\n",
      "8900\t0.1116\t3287.6326\n",
      "9000\t0.1162\t3283.2361\n",
      "9100\t0.1933\t3292.2098\n",
      "9200\t0.1359\t3281.9667\n",
      "9300\t0.0992\t3280.0216\n",
      "9400\t0.0939\t3301.02\n",
      "9500\t0.1789\t3295.4771\n",
      "9600\t0.1135\t3294.1977\n",
      "9700\t0.1231\t3302.1377\n",
      "9800\t0.1288\t3297.3199\n",
      "9900\t0.1759\t3261.4127\n",
      "10000\t0.1173\t3272.4575\n",
      "10100\t0.1466\t3283.4005\n",
      "10200\t0.1178\t3281.0342\n",
      "10300\t0.1025\t3269.5716\n",
      "10400\t0.1157\t3283.8918\n",
      "10500\t0.1204\t3277.0252\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "10600\t0.1348\t3269.8081\n",
      "10700\t0.1201\t3270.9214\n",
      "10800\t0.1293\t3276.1274\n",
      "10900\t0.1089\t3278.3244\n",
      "11000\t0.1204\t3276.1843\n",
      "11100\t0.144\t3276.0878\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "11200\t0.1164\t3279.0493\n",
      "11300\t0.1039\t3282.5159\n",
      "11400\t0.1027\t3283.2701\n",
      "11500\t0.1288\t3282.4866\n",
      "11600\t0.133\t3282.1126\n",
      "11700\t0.1361\t3281.6179\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "11800\t0.1342\t3280.7061\n",
      "11900\t0.1305\t3280.5385\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 243.7 seconds\n",
      "Best step:  9900\n",
      "Best Val Metric:  3261.4126503840903\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 189.17938232421875, std = 181.18861389160156\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.6802\t29280.3575\n",
      "200\t0.555\t24111.7844\n",
      "300\t0.46\t19604.3759\n",
      "400\t0.4305\t18563.8466\n",
      "500\t0.4451\t17458.8062\n",
      "600\t0.4079\t16374.3453\n",
      "700\t0.3938\t13843.3174\n",
      "800\t0.3263\t13776.464\n",
      "900\t0.2739\t13933.9603\n",
      "1000\t0.2635\t12902.716\n",
      "1100\t0.2284\t11389.5853\n",
      "1200\t0.1862\t10601.1293\n",
      "1300\t0.2423\t9652.8387\n",
      "1400\t0.1951\t9038.7915\n",
      "1500\t0.2472\t8755.9468\n",
      "1600\t0.1728\t8544.6391\n",
      "1700\t0.2781\t8444.6212\n",
      "1800\t0.203\t8465.7427\n",
      "1900\t0.2497\t7974.8846\n",
      "2000\t0.2264\t7475.6328\n",
      "2100\t0.208\t7217.1515\n",
      "2200\t0.2278\t6841.4621\n",
      "2300\t0.1578\t6515.3036\n",
      "2400\t0.2006\t6290.1585\n",
      "2500\t0.1667\t6084.6149\n",
      "2600\t0.182\t5864.9146\n",
      "2700\t0.2134\t5554.4235\n",
      "2800\t0.1734\t5390.1695\n",
      "2900\t0.2282\t5297.3917\n",
      "3000\t0.1439\t5206.8217\n",
      "3100\t0.2248\t5063.0341\n",
      "3200\t0.1331\t5026.0226\n",
      "3300\t0.1623\t5010.9427\n",
      "3400\t0.1974\t4867.9921\n",
      "3500\t0.1658\t4832.64\n",
      "3600\t0.1561\t4815.4384\n",
      "3700\t0.1941\t4787.8106\n",
      "3800\t0.1308\t4726.7503\n",
      "3900\t0.2273\t4712.073\n",
      "4000\t0.1567\t4693.7649\n",
      "4100\t0.23\t4716.0912\n",
      "4200\t0.1648\t4642.6499\n",
      "4300\t0.1842\t4630.7321\n",
      "4400\t0.2075\t4587.5944\n",
      "4500\t0.2915\t4532.3394\n",
      "4600\t0.1399\t4496.6061\n",
      "4700\t0.1383\t4485.5875\n",
      "4800\t0.1305\t4490.0015\n",
      "4900\t0.1495\t4509.9137\n",
      "5000\t0.1859\t4532.4137\n",
      "5100\t0.1299\t4509.65\n",
      "5200\t0.1529\t4482.2003\n",
      "5300\t0.1855\t4552.9853\n",
      "5400\t0.1757\t4589.4044\n",
      "5500\t0.1509\t4605.2646\n",
      "5600\t0.164\t4536.8801\n",
      "5700\t0.2282\t4518.8933\n",
      "5800\t0.2011\t4391.5701\n",
      "5900\t0.166\t4297.3864\n",
      "6000\t0.1952\t4294.3377\n",
      "6100\t0.1423\t4310.1783\n",
      "6200\t0.143\t4252.0977\n",
      "6300\t0.1502\t4246.1114\n",
      "6400\t0.1473\t4206.7948\n",
      "6500\t0.1439\t4169.956\n",
      "6600\t0.1639\t4186.6571\n",
      "6700\t0.1722\t4174.2174\n",
      "6800\t0.1692\t4136.3928\n",
      "6900\t0.179\t4074.9578\n",
      "7000\t0.1722\t4063.9511\n",
      "7100\t0.1358\t4069.0867\n",
      "7200\t0.1497\t4054.2421\n",
      "7300\t0.1378\t4186.5681\n",
      "7400\t0.206\t4168.5437\n",
      "7500\t0.2013\t4152.2892\n",
      "7600\t0.1513\t4050.8446\n",
      "7700\t0.1254\t4028.1357\n",
      "7800\t0.1327\t4035.2772\n",
      "7900\t0.1158\t3994.5026\n",
      "8000\t0.2266\t3979.4684\n",
      "8100\t0.1144\t4011.5427\n",
      "8200\t0.189\t4013.6971\n",
      "8300\t0.18\t3970.2627\n",
      "8400\t0.2088\t4002.1333\n",
      "8500\t0.1305\t3974.4877\n",
      "8600\t0.1553\t4066.3443\n",
      "8700\t0.1123\t4055.1422\n",
      "8800\t0.1486\t4053.3791\n",
      "8900\t0.1751\t3966.8046\n",
      "9000\t0.15\t3924.1463\n",
      "9100\t0.1146\t3867.533\n",
      "9200\t0.1349\t3896.848\n",
      "9300\t0.1389\t3887.1992\n",
      "9400\t0.1354\t3904.9356\n",
      "9500\t0.1503\t3919.9884\n",
      "9600\t0.1492\t4015.8623\n",
      "9700\t0.127\t4121.2371\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "9800\t0.1327\t4097.1535\n",
      "9900\t0.1264\t4230.0311\n",
      "10000\t0.128\t4237.7075\n",
      "10100\t0.2051\t4204.7065\n",
      "10200\t0.1176\t4204.2373\n",
      "10300\t0.1255\t4205.6027\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "10400\t0.1249\t4206.7743\n",
      "10500\t0.1171\t4190.4044\n",
      "10600\t0.1145\t4193.3532\n",
      "10700\t0.1239\t4192.9663\n",
      "10800\t0.1165\t4190.3652\n",
      "10900\t0.1526\t4184.2987\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "11000\t0.1433\t4183.4074\n",
      "11100\t0.1274\t4175.365\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 227.6 seconds\n",
      "Best step:  9100\n",
      "Best Val Metric:  3867.533028539665\n",
      "Load the best checkpoint.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.0026\t-0.9968\n",
      "200\t0.0078\t-0.9987\n",
      "300\t0.0077\t-0.9988\n",
      "400\t0.0008\t-0.9988\n",
      "500\t0.0015\t-0.9987\n",
      "600\t0.003\t-0.9987\n",
      "700\t0.0008\t-0.9984\n",
      "800\t0.0064\t-0.9985\n",
      "900\t0.001\t-0.9986\n",
      "1000\t0.0016\t-0.9987\n",
      "1100\t0.0028\t-0.9986\n",
      "1200\t0.0049\t-0.9986\n",
      "1300\t0.0037\t-0.9986\n",
      "1400\t0.0017\t-0.9985\n",
      "1500\t0.0038\t-0.9985\n",
      "1600\t0.0052\t-0.9985\n",
      "1700\t0.002\t-0.9986\n",
      "1800\t0.0049\t-0.9986\n",
      "1900\t0.0108\t-0.9987\n",
      "2000\t0.008\t-0.9986\n",
      "2100\t0.007\t-0.9986\n",
      "2200\t0.001\t-0.9986\n",
      "2300\t0.0009\t-0.9986\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "2400\t0.0007\t-0.9986\n",
      "2500\t0.0017\t-0.9986\n",
      "2600\t0.0059\t-0.9987\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "2700\t0.0048\t-0.9987\n",
      "2800\t0.0005\t-0.9987\n",
      "2900\t0.001\t-0.9987\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "3000\t0.0005\t-0.9987\n",
      "3100\t0.005\t-0.9987\n",
      "3200\t0.0013\t-0.9987\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "3300\t0.0039\t-0.9987\n",
      "3400\t0.008\t-0.9987\n",
      "3500\t0.0006\t-0.9987\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "3600\t0.0047\t-0.9987\n",
      "3700\t0.0045\t-0.9987\n",
      "3800\t0.0025\t-0.9987\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "3900\t0.0087\t-0.9987\n",
      "4000\t0.0014\t-0.9987\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 87.6 seconds\n",
      "Best step:  400\n",
      "Best Val Metric:  -0.9988181258872378\n",
      "Load the best checkpoint.\n",
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.0094\t-0.9634\n",
      "200\t0.002\t-0.9757\n",
      "300\t0.0039\t-0.981\n",
      "400\t0.0005\t-0.9825\n",
      "500\t0.0021\t-0.9834\n",
      "600\t0.0001\t-0.9832\n",
      "700\t0.0136\t-0.9841\n",
      "800\t0.0012\t-0.9848\n",
      "900\t0.0042\t-0.9828\n",
      "1000\t0.0028\t-0.9827\n",
      "1100\t0.0006\t-0.9817\n",
      "1200\t0.0009\t-0.9819\n",
      "1300\t0.0033\t-0.982\n",
      "1400\t0.0013\t-0.982\n",
      "1500\t0.0007\t-0.9824\n",
      "1600\t0.0011\t-0.9826\n",
      "1700\t0.0013\t-0.9828\n",
      "1800\t0.0089\t-0.9831\n",
      "1900\t0.0005\t-0.9834\n",
      "2000\t0.0029\t-0.9836\n",
      "2100\t0.0002\t-0.9837\n",
      "2200\t0.0001\t-0.9841\n",
      "2300\t0.0011\t-0.984\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "2400\t0.0038\t-0.984\n",
      "2500\t0.0049\t-0.9839\n",
      "2600\t0.0005\t-0.9839\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "2700\t0.0033\t-0.9835\n",
      "2800\t0.0007\t-0.9837\n",
      "2900\t0.0007\t-0.9839\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "3000\t0.0017\t-0.984\n",
      "3100\t0.0024\t-0.9839\n",
      "3200\t0.0025\t-0.9839\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "3300\t0.0073\t-0.9839\n",
      "3400\t0.0051\t-0.9839\n",
      "3500\t0.0005\t-0.9839\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "3600\t0.001\t-0.9839\n",
      "3700\t0.002\t-0.9839\n",
      "3800\t0.0057\t-0.9839\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "3900\t0.0038\t-0.9839\n",
      "4000\t0.0037\t-0.9839\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 87.4 seconds\n",
      "Best step:  800\n",
      "Best Val Metric:  -0.9847723004830193\n",
      "Load the best checkpoint.\n",
      "Steps\tTrain Err\tVal Metric (negative_auc)\n",
      "100\t0.0007\t-0.956\n",
      "200\t0.0065\t-0.9593\n",
      "300\t0.0012\t-0.9588\n",
      "400\t0.0008\t-0.9639\n",
      "500\t0.001\t-0.9668\n",
      "600\t0.0083\t-0.9694\n",
      "700\t0.0014\t-0.973\n",
      "800\t0.0054\t-0.9735\n",
      "900\t0.0019\t-0.9732\n",
      "1000\t0.0049\t-0.9742\n",
      "1100\t0.0025\t-0.9742\n",
      "1200\t0.0036\t-0.9742\n",
      "1300\t0.0024\t-0.9742\n",
      "1400\t0.0006\t-0.9744\n",
      "1500\t0.0034\t-0.9746\n",
      "1600\t0.0047\t-0.9746\n",
      "1700\t0.0004\t-0.9746\n",
      "1800\t0.0015\t-0.9748\n",
      "1900\t0.0003\t-0.9751\n",
      "2000\t0.0032\t-0.9755\n",
      "2100\t0.0043\t-0.9759\n",
      "2200\t0.0002\t-0.9767\n",
      "2300\t0.001\t-0.9771\n",
      "2400\t0.0003\t-0.9774\n",
      "2500\t0.0009\t-0.9777\n",
      "2600\t0.0017\t-0.9777\n",
      "2700\t0.0037\t-0.978\n",
      "2800\t0.0008\t-0.9785\n",
      "2900\t0.0002\t-0.979\n",
      "3000\t0.0002\t-0.9796\n",
      "3100\t0.0007\t-0.9807\n",
      "3200\t0.008\t-0.9814\n",
      "3300\t0.0005\t-0.9823\n",
      "3400\t0.0044\t-0.9832\n",
      "3500\t0.0007\t-0.9837\n",
      "3600\t0.0005\t-0.9842\n",
      "3700\t0.0046\t-0.9846\n",
      "3800\t0.0003\t-0.9851\n",
      "3900\t0.0015\t-0.9852\n",
      "4000\t0.0004\t-0.9857\n",
      "4100\t0.0021\t-0.9862\n",
      "4200\t0.0002\t-0.9863\n",
      "4300\t0.0007\t-0.9862\n",
      "4400\t0.0011\t-0.9866\n",
      "4500\t0.0002\t-0.9872\n",
      "4600\t0.0064\t-0.9877\n",
      "4700\t0.0001\t-0.9881\n",
      "4800\t0.0016\t-0.9885\n",
      "4900\t0.002\t-0.9882\n",
      "5000\t0.0\t-0.9878\n",
      "5100\t0.0035\t-0.9872\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "5200\t0.0002\t-0.9867\n",
      "5300\t0.006\t-0.986\n",
      "5400\t0.0003\t-0.9863\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "5500\t0.0033\t-0.9859\n",
      "5600\t0.0004\t-0.9857\n",
      "5700\t0.0001\t-0.9856\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "5800\t0.0001\t-0.9857\n",
      "5900\t0.0002\t-0.9857\n",
      "6000\t0.0003\t-0.9857\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "6100\t0.0023\t-0.9856\n",
      "6200\t0.0002\t-0.9856\n",
      "6300\t0.0005\t-0.9857\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "6400\t0.0007\t-0.9857\n",
      "6500\t0.0002\t-0.9857\n",
      "6600\t0.0001\t-0.9857\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "6700\t0.0016\t-0.9857\n",
      "6800\t0.0012\t-0.9857\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 151.1 seconds\n",
      "Best step:  4800\n",
      "Best Val Metric:  -0.9885193640607968\n",
      "Load the best checkpoint.\n"
     ]
    }
   ],
   "source": [
    "for dset in ['adult', 'mimic2', 'wine', 'bikeshare', 'credit']:\n",
    "    for model_name in ['ebm', 'xgb-gam', 'xgb', 'nodegam']:\n",
    "        for fold in [0, 1, 2]:\n",
    "            if any([(r['dataset'] == dset and r['model_name'] == model_name and r['fold'] == fold)\n",
    "                    for r in records]):\n",
    "                print(f'Already run {dset} {model_name} {fold}')\n",
    "                continue\n",
    "\n",
    "            try:\n",
    "                record = run(data_name=dset, model_name=model_name, fold=fold)\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                record = dict(model_name=model_name, dataset=dset, seed=seed, error_msg=str(e))\n",
    "            records.append(record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "50f648f7-5023-4b3c-9deb-f8b9b3d2a189",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalize y. mean = 1998.39208984375, std = 10.92832088470459\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t1.3973\t147.3397\n",
      "200\t0.822\t126.1152\n",
      "300\t0.6901\t118.5757\n",
      "400\t0.7585\t103.8585\n",
      "500\t0.8088\t101.5735\n",
      "600\t0.7972\t92.8928\n",
      "700\t0.719\t91.6918\n",
      "800\t0.6853\t88.1823\n",
      "900\t0.6953\t88.4449\n",
      "1000\t0.7137\t87.368\n",
      "1100\t0.7081\t85.6934\n",
      "1200\t0.6987\t84.8604\n",
      "1300\t0.7011\t83.5634\n",
      "1400\t0.6817\t82.9605\n",
      "1500\t0.6903\t82.4482\n",
      "1600\t0.7292\t82.1023\n",
      "1700\t0.7731\t82.1228\n",
      "1800\t0.7712\t81.7418\n",
      "1900\t0.7389\t81.6894\n",
      "2000\t0.7198\t81.6422\n",
      "2100\t0.7435\t81.4722\n",
      "2200\t0.6648\t81.4725\n",
      "2300\t0.6464\t81.4625\n",
      "2400\t0.7406\t81.4186\n",
      "2500\t0.6934\t81.2779\n",
      "2600\t0.7473\t81.2658\n",
      "2700\t0.6994\t81.1909\n",
      "2800\t0.686\t81.1997\n",
      "2900\t0.6941\t81.1207\n",
      "3000\t0.6923\t81.1089\n",
      "3100\t0.7375\t81.1294\n",
      "3200\t0.6944\t81.1409\n",
      "3300\t0.7176\t81.1783\n",
      "3400\t0.7032\t81.1642\n",
      "3500\t0.671\t81.0931\n",
      "3600\t0.6597\t81.0792\n",
      "3700\t0.6267\t81.1891\n",
      "3800\t0.6795\t81.0583\n",
      "3900\t0.5982\t80.9935\n",
      "4000\t0.6596\t80.9999\n",
      "4100\t0.6956\t80.9721\n",
      "4200\t0.7188\t80.9337\n",
      "4300\t0.6243\t80.9184\n",
      "4400\t0.7028\t80.9298\n",
      "4500\t0.7297\t80.8818\n",
      "4600\t0.6301\t80.8514\n",
      "4700\t0.6727\t80.9255\n",
      "4800\t0.778\t80.8155\n",
      "4900\t0.7468\t80.7767\n",
      "5000\t0.7075\t80.9041\n",
      "5100\t0.7215\t80.8706\n",
      "5200\t0.6338\t80.9333\n",
      "5300\t0.7538\t80.873\n",
      "5400\t0.6702\t80.8485\n",
      "5500\t0.7542\t80.8694\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "5600\t0.7192\t80.9034\n",
      "5700\t0.6522\t80.8916\n",
      "5800\t0.6888\t80.917\n",
      "5900\t0.667\t80.861\n",
      "6000\t0.7061\t80.6974\n",
      "6100\t0.7094\t80.6823\n",
      "6200\t0.6969\t80.6524\n",
      "6300\t0.6562\t80.6431\n",
      "6400\t0.6745\t80.6381\n",
      "6500\t0.6667\t80.6181\n",
      "6600\t0.6617\t80.6004\n",
      "6700\t0.6601\t80.606\n",
      "6800\t0.6317\t80.5921\n",
      "6900\t0.6732\t80.5699\n",
      "7000\t0.7039\t80.5583\n",
      "7100\t0.6883\t80.5759\n",
      "7200\t0.7027\t80.5828\n",
      "7300\t0.6328\t80.6042\n",
      "7400\t0.6319\t80.6034\n",
      "7500\t0.6605\t80.5719\n",
      "7600\t0.6996\t80.5358\n",
      "7700\t0.6779\t80.5401\n",
      "7800\t0.7218\t80.5229\n",
      "7900\t0.6433\t80.5518\n",
      "8000\t0.6947\t80.5694\n",
      "8100\t0.7631\t80.55\n",
      "8200\t0.6743\t80.5224\n",
      "8300\t0.6729\t80.5227\n",
      "8400\t0.7662\t80.5056\n",
      "8500\t0.6809\t80.497\n",
      "8600\t0.6196\t80.5385\n",
      "8700\t0.6679\t80.5341\n",
      "8800\t0.5923\t80.5667\n",
      "8900\t0.6916\t80.5903\n",
      "9000\t0.6476\t80.6055\n",
      "9100\t0.7746\t80.5542\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "9200\t0.6907\t80.5619\n",
      "9300\t0.7496\t80.5322\n",
      "9400\t0.6881\t80.5294\n",
      "9500\t0.7287\t80.5178\n",
      "9600\t0.6453\t80.4968\n",
      "9700\t0.6946\t80.4844\n",
      "9800\t0.6728\t80.4883\n",
      "9900\t0.6475\t80.4866\n",
      "10000\t0.785\t80.4874\n",
      "10100\t0.6711\t80.4917\n",
      "10200\t0.6938\t80.4891\n",
      "10300\t0.6714\t80.4899\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "10400\t0.6728\t80.4914\n",
      "10500\t0.6741\t80.4947\n",
      "10600\t0.689\t80.4905\n",
      "10700\t0.7189\t80.4971\n",
      "10800\t0.7038\t80.4932\n",
      "10900\t0.7126\t80.4901\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "11000\t0.7132\t80.4887\n",
      "11100\t0.6735\t80.4884\n",
      "11200\t0.6382\t80.4891\n",
      "11300\t0.641\t80.4868\n",
      "11400\t0.6352\t80.4851\n",
      "11500\t0.6212\t80.483\n",
      "11600\t0.7029\t80.4832\n",
      "11700\t0.6816\t80.4831\n",
      "11800\t0.6628\t80.4839\n",
      "11900\t0.6465\t80.486\n",
      "12000\t0.6582\t80.4891\n",
      "12100\t0.6753\t80.4896\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "12200\t0.7129\t80.4895\n",
      "12300\t0.6811\t80.489\n",
      "12400\t0.6688\t80.4879\n",
      "12500\t0.6565\t80.4863\n",
      "12600\t0.6998\t80.4852\n",
      "12700\t0.6424\t80.4848\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "12800\t0.6586\t80.4846\n",
      "12900\t0.704\t80.4844\n",
      "13000\t0.7134\t80.4844\n",
      "13100\t0.6562\t80.4845\n",
      "13200\t0.7038\t80.4844\n",
      "13300\t0.6898\t80.4844\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "13400\t0.6544\t80.4845\n",
      "13500\t0.7129\t80.4845\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 322.9 seconds\n",
      "Best step:  11500\n",
      "Best Val Metric:  80.48296215574156\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 1998.39208984375, std = 10.92832088470459\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.863\t181.1611\n",
      "200\t0.6966\t145.369\n",
      "300\t0.7077\t135.1886\n",
      "400\t0.8755\t116.8785\n",
      "500\t0.6851\t110.2451\n",
      "600\t0.6978\t99.4987\n",
      "700\t0.6648\t95.719\n",
      "800\t0.7072\t88.5505\n",
      "900\t0.6727\t88.0267\n",
      "1000\t0.6763\t87.6748\n",
      "1100\t0.6973\t85.7443\n",
      "1200\t0.743\t84.6847\n",
      "1300\t0.7276\t84.4199\n",
      "1400\t0.6623\t83.6661\n",
      "1500\t0.6947\t82.9209\n",
      "1600\t0.7438\t82.6279\n",
      "1700\t0.6861\t82.37\n",
      "1800\t0.691\t82.2165\n",
      "1900\t0.6825\t82.1304\n",
      "2000\t0.684\t82.0081\n",
      "2100\t0.7211\t82.0478\n",
      "2200\t0.6575\t81.9039\n",
      "2300\t0.6393\t81.9197\n",
      "2400\t0.7211\t81.7556\n",
      "2500\t0.6961\t81.7674\n",
      "2600\t0.7118\t81.6595\n",
      "2700\t0.7596\t81.6494\n",
      "2800\t0.7019\t81.6268\n",
      "2900\t0.6678\t81.5559\n",
      "3000\t0.759\t81.4426\n",
      "3100\t0.6666\t81.3742\n",
      "3200\t0.7288\t81.3683\n",
      "3300\t0.6992\t81.3444\n",
      "3400\t0.7233\t81.3007\n",
      "3500\t0.7691\t81.3391\n",
      "3600\t0.7224\t81.3445\n",
      "3700\t0.6586\t81.2465\n",
      "3800\t0.7544\t81.2856\n",
      "3900\t0.6927\t81.3578\n",
      "4000\t0.7053\t81.3758\n",
      "4100\t0.6858\t81.3351\n",
      "4200\t0.6209\t81.2647\n",
      "4300\t0.7352\t81.2086\n",
      "4400\t0.6608\t81.1812\n",
      "4500\t0.6888\t81.1612\n",
      "4600\t0.6834\t81.1278\n",
      "4700\t0.639\t81.1279\n",
      "4800\t0.6933\t81.1431\n",
      "4900\t0.6671\t81.1573\n",
      "5000\t0.605\t81.1224\n",
      "5100\t0.7023\t81.1153\n",
      "5200\t0.7009\t81.256\n",
      "5300\t0.6835\t81.546\n",
      "5400\t0.7332\t81.2771\n",
      "5500\t0.6848\t81.3486\n",
      "5600\t0.6296\t81.2291\n",
      "5700\t0.7684\t81.1691\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "5800\t0.6785\t81.0579\n",
      "5900\t0.689\t81.0283\n",
      "6000\t0.7077\t80.9968\n",
      "6100\t0.6896\t80.916\n",
      "6200\t0.684\t80.851\n",
      "6300\t0.6429\t80.8588\n",
      "6400\t0.6349\t80.833\n",
      "6500\t0.6711\t80.8373\n",
      "6600\t0.7137\t80.8623\n",
      "6700\t0.6764\t80.8666\n",
      "6800\t0.6737\t80.858\n",
      "6900\t0.7636\t80.854\n",
      "7000\t0.6726\t80.8277\n",
      "7100\t0.7408\t80.807\n",
      "7200\t0.598\t80.7949\n",
      "7300\t0.7168\t80.7868\n",
      "7400\t0.6908\t80.781\n",
      "7500\t0.6969\t80.7944\n",
      "7600\t0.6278\t80.7959\n",
      "7700\t0.6864\t80.8023\n",
      "7800\t0.6574\t80.7977\n",
      "7900\t0.6072\t80.8463\n",
      "8000\t0.7155\t80.8261\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "8100\t0.7012\t80.8302\n",
      "8200\t0.6928\t80.8107\n",
      "8300\t0.6432\t80.7965\n",
      "8400\t0.6844\t80.7488\n",
      "8500\t0.7156\t80.7556\n",
      "8600\t0.6315\t80.7503\n",
      "8700\t0.6412\t80.7419\n",
      "8800\t0.6664\t80.7347\n",
      "8900\t0.7074\t80.7301\n",
      "9000\t0.6943\t80.7297\n",
      "9100\t0.7028\t80.7271\n",
      "9200\t0.6142\t80.7371\n",
      "9300\t0.7068\t80.7457\n",
      "9400\t0.6752\t80.7425\n",
      "9500\t0.7284\t80.7305\n",
      "9600\t0.6828\t80.7346\n",
      "9700\t0.7697\t80.7354\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "9800\t0.7079\t80.7318\n",
      "9900\t0.701\t80.7359\n",
      "10000\t0.6975\t80.7414\n",
      "10100\t0.6878\t80.737\n",
      "10200\t0.5812\t80.7345\n",
      "10300\t0.6946\t80.7285\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "10400\t0.6418\t80.7268\n",
      "10500\t0.6663\t80.724\n",
      "10600\t0.6597\t80.7237\n",
      "10700\t0.6758\t80.7205\n",
      "10800\t0.6677\t80.7213\n",
      "10900\t0.6089\t80.7215\n",
      "11000\t0.6755\t80.7224\n",
      "11100\t0.679\t80.7226\n",
      "11200\t0.7122\t80.7237\n",
      "11300\t0.6734\t80.724\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "11400\t0.6219\t80.7247\n",
      "11500\t0.6166\t80.7246\n",
      "11600\t0.6561\t80.7248\n",
      "11700\t0.6292\t80.7247\n",
      "11800\t0.6901\t80.7252\n",
      "11900\t0.6669\t80.7257\n",
      "LR: 3.20e-06 -> 1.00e-06\n",
      "12000\t0.6947\t80.7264\n",
      "12100\t0.6996\t80.727\n",
      "12200\t0.6882\t80.7271\n",
      "12300\t0.7442\t80.727\n",
      "12400\t0.6778\t80.7268\n",
      "12500\t0.7083\t80.7265\n",
      "LR: 1.00e-06 -> 1.00e-06\n",
      "12600\t0.7735\t80.7262\n",
      "12700\t0.6612\t80.726\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 306.1 seconds\n",
      "Best step:  10700\n",
      "Best Val Metric:  80.72049143887635\n",
      "Load the best checkpoint.\n",
      "Normalize y. mean = 1998.39208984375, std = 10.92832088470459\n",
      "Steps\tTrain Err\tVal Metric (mse)\n",
      "100\t0.9042\t180.7305\n",
      "200\t0.7122\t145.014\n",
      "300\t0.7023\t128.9641\n",
      "400\t0.658\t108.0891\n",
      "500\t0.6518\t105.3884\n",
      "600\t0.6441\t92.5638\n",
      "700\t0.7065\t88.9944\n",
      "800\t0.6581\t89.1551\n",
      "900\t0.656\t87.0374\n",
      "1000\t0.735\t84.9291\n",
      "1100\t0.7037\t84.5827\n",
      "1200\t0.7345\t83.777\n",
      "1300\t0.6642\t83.4811\n",
      "1400\t0.7075\t83.3459\n",
      "1500\t0.6544\t82.8319\n",
      "1600\t0.7096\t82.4976\n",
      "1700\t0.6515\t82.3514\n",
      "1800\t0.699\t82.1337\n",
      "1900\t0.6768\t81.841\n",
      "2000\t0.7432\t81.7673\n",
      "2100\t0.705\t81.7013\n",
      "2200\t0.7201\t81.6255\n",
      "2300\t0.6838\t81.5696\n",
      "2400\t0.669\t81.5278\n",
      "2500\t0.6741\t81.587\n",
      "2600\t0.6726\t81.5356\n",
      "2700\t0.7351\t81.5345\n",
      "2800\t0.7124\t81.5114\n",
      "2900\t0.6935\t81.5651\n",
      "3000\t0.6582\t81.4866\n",
      "3100\t0.6358\t81.4533\n",
      "3200\t0.6992\t81.4116\n",
      "3300\t0.7248\t81.3605\n",
      "3400\t0.7531\t81.2924\n",
      "3500\t0.719\t81.2869\n",
      "3600\t0.7612\t81.248\n",
      "3700\t0.6504\t81.228\n",
      "3800\t0.6356\t81.2191\n",
      "3900\t0.6785\t81.1513\n",
      "4000\t0.6839\t81.2188\n",
      "4100\t0.687\t81.186\n",
      "4200\t0.7515\t81.2677\n",
      "4300\t0.7402\t81.2721\n",
      "4400\t0.6833\t81.2545\n",
      "4500\t0.657\t81.2045\n",
      "LR: 1.00e-02 -> 2.00e-03\n",
      "4600\t0.6938\t81.1941\n",
      "4700\t0.6832\t81.0418\n",
      "4800\t0.7412\t81.0089\n",
      "4900\t0.6691\t80.9749\n",
      "5000\t0.6673\t80.914\n",
      "5100\t0.7136\t80.891\n",
      "5200\t0.6692\t80.8737\n",
      "5300\t0.7854\t80.8755\n",
      "5400\t0.687\t80.8626\n",
      "5500\t0.7325\t80.841\n",
      "5600\t0.6899\t80.8348\n",
      "5700\t0.6562\t80.886\n",
      "5800\t0.6809\t80.8837\n",
      "5900\t0.6737\t80.9065\n",
      "6000\t0.723\t80.9101\n",
      "6100\t0.6602\t80.9093\n",
      "6200\t0.7148\t80.8302\n",
      "6300\t0.6547\t80.8254\n",
      "6400\t0.6609\t80.7975\n",
      "6500\t0.7318\t80.8153\n",
      "6600\t0.7016\t80.8086\n",
      "6700\t0.6621\t80.8089\n",
      "6800\t0.6856\t80.7933\n",
      "6900\t0.7013\t80.7998\n",
      "7000\t0.6186\t80.7567\n",
      "7100\t0.6705\t80.7541\n",
      "7200\t0.672\t80.7545\n",
      "7300\t0.6724\t80.7609\n",
      "7400\t0.6632\t80.7524\n",
      "7500\t0.6383\t80.7663\n",
      "7600\t0.6712\t80.7486\n",
      "7700\t0.7211\t80.7336\n",
      "7800\t0.6408\t80.7277\n",
      "7900\t0.777\t80.7097\n",
      "8000\t0.6425\t80.7172\n",
      "8100\t0.7324\t80.7125\n",
      "8200\t0.6951\t80.6941\n",
      "8300\t0.7164\t80.6671\n",
      "8400\t0.7002\t80.6666\n",
      "8500\t0.7163\t80.642\n",
      "8600\t0.6623\t80.6308\n",
      "8700\t0.655\t80.6599\n",
      "8800\t0.6858\t80.6715\n",
      "8900\t0.7013\t80.6829\n",
      "9000\t0.6676\t80.6722\n",
      "9100\t0.7149\t80.6545\n",
      "9200\t0.6295\t80.6387\n",
      "LR: 2.00e-03 -> 4.00e-04\n",
      "9300\t0.6618\t80.6386\n",
      "9400\t0.6724\t80.6357\n",
      "9500\t0.6898\t80.6414\n",
      "9600\t0.6439\t80.6412\n",
      "9700\t0.6912\t80.632\n",
      "9800\t0.6853\t80.6225\n",
      "9900\t0.6452\t80.6154\n",
      "10000\t0.6851\t80.6193\n",
      "10100\t0.6306\t80.6256\n",
      "10200\t0.6896\t80.6214\n",
      "10300\t0.6551\t80.6321\n",
      "10400\t0.6446\t80.6425\n",
      "10500\t0.7414\t80.6381\n",
      "LR: 4.00e-04 -> 8.00e-05\n",
      "10600\t0.6678\t80.642\n",
      "10700\t0.6908\t80.6563\n",
      "10800\t0.6542\t80.6495\n",
      "10900\t0.6912\t80.6465\n",
      "11000\t0.6088\t80.6415\n",
      "11100\t0.6384\t80.6323\n",
      "LR: 8.00e-05 -> 1.60e-05\n",
      "11200\t0.634\t80.6296\n",
      "11300\t0.6944\t80.6271\n",
      "11400\t0.6983\t80.6247\n",
      "11500\t0.7081\t80.6212\n",
      "11600\t0.6581\t80.6208\n",
      "11700\t0.6169\t80.619\n",
      "LR: 1.60e-05 -> 3.20e-06\n",
      "11800\t0.7083\t80.6187\n",
      "11900\t0.6578\t80.6183\n",
      "BREAK. There is no improvment for 2000 steps\n",
      "Total training time: 285.8 seconds\n",
      "Best step:  9900\n",
      "Best Val Metric:  80.61535867746969\n",
      "Load the best checkpoint.\n"
     ]
    }
   ],
   "source": [
    "for dset in ['year']:\n",
    "    for model_name in ['ebm', 'xgb-gam', 'xgb', 'nodegam']:\n",
    "        for seed in [0, 1, 2]:\n",
    "            if any([(r['dataset'] == dset and r['model_name'] == model_name and r['seed'] == seed)\n",
    "                    for r in records]):\n",
    "                print(f'Already run {dset} {model_name} {fold}')\n",
    "                continue\n",
    "\n",
    "            try:\n",
    "                record = run(data_name=dset, model_name=model_name, seed=seed)\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                record = dict(model_name=model_name, dataset=dset, seed=seed, error_msg=str(e))\n",
    "            records.append(record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab0da051-9e73-4df2-a4a7-2bff2dab173e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>model_name</th>\n",
       "      <th>fold</th>\n",
       "      <th>seed</th>\n",
       "      <th>test_perf</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>adult</td>\n",
       "      <td>ebm</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.929210</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>adult</td>\n",
       "      <td>ebm</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>0.923043</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>adult</td>\n",
       "      <td>ebm</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>0.927641</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>adult</td>\n",
       "      <td>xgb-gam</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.924382</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>adult</td>\n",
       "      <td>xgb-gam</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>0.922595</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>year</td>\n",
       "      <td>xgb</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9.048994</td>\n",
       "      <td>537.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>year</td>\n",
       "      <td>xgb</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9.048994</td>\n",
       "      <td>538.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>year</td>\n",
       "      <td>nodegam</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.009680</td>\n",
       "      <td>337.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>year</td>\n",
       "      <td>nodegam</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9.018009</td>\n",
       "      <td>318.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>year</td>\n",
       "      <td>nodegam</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>9.011756</td>\n",
       "      <td>298.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>72 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   dataset model_name  fold  seed  test_perf   time\n",
       "0    adult        ebm     0    31   0.929210   24.0\n",
       "1    adult        ebm     1    31   0.923043   11.0\n",
       "2    adult        ebm     2    31   0.927641   11.0\n",
       "3    adult    xgb-gam     0    31   0.924382    6.0\n",
       "4    adult    xgb-gam     1    31   0.922595    6.0\n",
       "..     ...        ...   ...   ...        ...    ...\n",
       "67    year        xgb     0     1   9.048994  537.0\n",
       "68    year        xgb     0     2   9.048994  538.0\n",
       "69    year    nodegam     0     0   9.009680  337.0\n",
       "70    year    nodegam     0     1   9.018009  318.0\n",
       "71    year    nodegam     0     2   9.011756  298.0\n",
       "\n",
       "[72 rows x 6 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(records)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a316fada-5d1f-47af-bcfe-bc173136c84a",
   "metadata": {},
   "outputs": [],
   "source": [
    "perf_df = df.groupby(['dataset', 'model_name']).agg(\n",
    "    {'test_perf': ['mean', 'std'], 'time': ['mean', 'std']}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "433d6d29-30f4-47b7-a468-1c2e684248e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">test_perf</th>\n",
       "      <th colspan=\"2\" halign=\"left\">time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>model_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">adult</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.926631</td>\n",
       "      <td>0.003205</td>\n",
       "      <td>15.333333</td>\n",
       "      <td>7.505553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.915616</td>\n",
       "      <td>0.002189</td>\n",
       "      <td>196.333333</td>\n",
       "      <td>55.509759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.927330</td>\n",
       "      <td>0.002119</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.924643</td>\n",
       "      <td>0.002190</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">bikeshare</th>\n",
       "      <th>ebm</th>\n",
       "      <td>55.675896</td>\n",
       "      <td>0.327145</td>\n",
       "      <td>15.333333</td>\n",
       "      <td>2.516611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>57.438405</td>\n",
       "      <td>3.898916</td>\n",
       "      <td>223.333333</td>\n",
       "      <td>23.352373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>45.212191</td>\n",
       "      <td>1.253863</td>\n",
       "      <td>1.666667</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>101.093015</td>\n",
       "      <td>0.946320</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">credit</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.983871</td>\n",
       "      <td>0.006659</td>\n",
       "      <td>36.666667</td>\n",
       "      <td>2.081666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.989025</td>\n",
       "      <td>0.008174</td>\n",
       "      <td>112.666667</td>\n",
       "      <td>35.809682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.984117</td>\n",
       "      <td>0.010005</td>\n",
       "      <td>15.666667</td>\n",
       "      <td>1.527525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.985473</td>\n",
       "      <td>0.008172</td>\n",
       "      <td>25.666667</td>\n",
       "      <td>7.023769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">mimic2</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.841761</td>\n",
       "      <td>0.019344</td>\n",
       "      <td>5.666667</td>\n",
       "      <td>2.081666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.844460</td>\n",
       "      <td>0.017504</td>\n",
       "      <td>105.333333</td>\n",
       "      <td>14.153916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.845281</td>\n",
       "      <td>0.018835</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.833436</td>\n",
       "      <td>0.019542</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">wine</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.690175</td>\n",
       "      <td>0.010543</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>2.081666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.705424</td>\n",
       "      <td>0.011629</td>\n",
       "      <td>157.000000</td>\n",
       "      <td>85.632938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.681934</td>\n",
       "      <td>0.022948</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.713398</td>\n",
       "      <td>0.006168</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">year</th>\n",
       "      <th>ebm</th>\n",
       "      <td>9.204122</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>501.000000</td>\n",
       "      <td>7.937254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>9.013148</td>\n",
       "      <td>0.004336</td>\n",
       "      <td>317.666667</td>\n",
       "      <td>19.502137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>9.048994</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>537.333333</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>9.256624</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>376.333333</td>\n",
       "      <td>0.577350</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       test_perf                  time           \n",
       "                            mean       std        mean        std\n",
       "dataset   model_name                                             \n",
       "adult     ebm           0.926631  0.003205   15.333333   7.505553\n",
       "          nodegam       0.915616  0.002189  196.333333  55.509759\n",
       "          xgb           0.927330  0.002119    1.000000   0.000000\n",
       "          xgb-gam       0.924643  0.002190    6.000000   0.000000\n",
       "bikeshare ebm          55.675896  0.327145   15.333333   2.516611\n",
       "          nodegam      57.438405  3.898916  223.333333  23.352373\n",
       "          xgb          45.212191  1.253863    1.666667   0.577350\n",
       "          xgb-gam     101.093015  0.946320    0.666667   0.577350\n",
       "credit    ebm           0.983871  0.006659   36.666667   2.081666\n",
       "          nodegam       0.989025  0.008174  112.666667  35.809682\n",
       "          xgb           0.984117  0.010005   15.666667   1.527525\n",
       "          xgb-gam       0.985473  0.008172   25.666667   7.023769\n",
       "mimic2    ebm           0.841761  0.019344    5.666667   2.081666\n",
       "          nodegam       0.844460  0.017504  105.333333  14.153916\n",
       "          xgb           0.845281  0.018835    0.666667   0.577350\n",
       "          xgb-gam       0.833436  0.019542    0.333333   0.577350\n",
       "wine      ebm           0.690175  0.010543    3.666667   2.081666\n",
       "          nodegam       0.705424  0.011629  157.000000  85.632938\n",
       "          xgb           0.681934  0.022948    0.000000   0.000000\n",
       "          xgb-gam       0.713398  0.006168    0.000000   0.000000\n",
       "year      ebm           9.204122  0.000000  501.000000   7.937254\n",
       "          nodegam       9.013148  0.004336  317.666667  19.502137\n",
       "          xgb           9.048994  0.000000  537.333333   0.577350\n",
       "          xgb-gam       9.256624  0.000000  376.333333   0.577350"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5afdf859-7e9b-4e33-9ef7-b3b7a68e3521",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>model_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">adult</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.926631</td>\n",
       "      <td>0.003205</td>\n",
       "      <td>0.927 ± 0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.915616</td>\n",
       "      <td>0.002189</td>\n",
       "      <td>0.916 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.927330</td>\n",
       "      <td>0.002119</td>\n",
       "      <td>0.927 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.924643</td>\n",
       "      <td>0.002190</td>\n",
       "      <td>0.925 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">bikeshare</th>\n",
       "      <th>ebm</th>\n",
       "      <td>55.675896</td>\n",
       "      <td>0.327145</td>\n",
       "      <td>55.676 ± 0.327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>57.438405</td>\n",
       "      <td>3.898916</td>\n",
       "      <td>57.438 ± 3.899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>45.212191</td>\n",
       "      <td>1.253863</td>\n",
       "      <td>45.212 ± 1.254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>101.093015</td>\n",
       "      <td>0.946320</td>\n",
       "      <td>101.093 ± 0.946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">credit</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.983871</td>\n",
       "      <td>0.006659</td>\n",
       "      <td>0.984 ± 0.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.989025</td>\n",
       "      <td>0.008174</td>\n",
       "      <td>0.989 ± 0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.984117</td>\n",
       "      <td>0.010005</td>\n",
       "      <td>0.984 ± 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.985473</td>\n",
       "      <td>0.008172</td>\n",
       "      <td>0.985 ± 0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">mimic2</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.841761</td>\n",
       "      <td>0.019344</td>\n",
       "      <td>0.842 ± 0.019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.844460</td>\n",
       "      <td>0.017504</td>\n",
       "      <td>0.844 ± 0.018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.845281</td>\n",
       "      <td>0.018835</td>\n",
       "      <td>0.845 ± 0.019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.833436</td>\n",
       "      <td>0.019542</td>\n",
       "      <td>0.833 ± 0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">wine</th>\n",
       "      <th>ebm</th>\n",
       "      <td>0.690175</td>\n",
       "      <td>0.010543</td>\n",
       "      <td>0.69 ± 0.011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>0.705424</td>\n",
       "      <td>0.011629</td>\n",
       "      <td>0.705 ± 0.012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.681934</td>\n",
       "      <td>0.022948</td>\n",
       "      <td>0.682 ± 0.023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.713398</td>\n",
       "      <td>0.006168</td>\n",
       "      <td>0.713 ± 0.006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">year</th>\n",
       "      <th>ebm</th>\n",
       "      <td>9.204122</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.204 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>9.013148</td>\n",
       "      <td>0.004336</td>\n",
       "      <td>9.013 ± 0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>9.048994</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.049 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>9.256624</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.257 ± 0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            mean       std          summary\n",
       "dataset   model_name                                       \n",
       "adult     ebm           0.926631  0.003205    0.927 ± 0.003\n",
       "          nodegam       0.915616  0.002189    0.916 ± 0.002\n",
       "          xgb           0.927330  0.002119    0.927 ± 0.002\n",
       "          xgb-gam       0.924643  0.002190    0.925 ± 0.002\n",
       "bikeshare ebm          55.675896  0.327145   55.676 ± 0.327\n",
       "          nodegam      57.438405  3.898916   57.438 ± 3.899\n",
       "          xgb          45.212191  1.253863   45.212 ± 1.254\n",
       "          xgb-gam     101.093015  0.946320  101.093 ± 0.946\n",
       "credit    ebm           0.983871  0.006659    0.984 ± 0.007\n",
       "          nodegam       0.989025  0.008174    0.989 ± 0.008\n",
       "          xgb           0.984117  0.010005     0.984 ± 0.01\n",
       "          xgb-gam       0.985473  0.008172    0.985 ± 0.008\n",
       "mimic2    ebm           0.841761  0.019344    0.842 ± 0.019\n",
       "          nodegam       0.844460  0.017504    0.844 ± 0.018\n",
       "          xgb           0.845281  0.018835    0.845 ± 0.019\n",
       "          xgb-gam       0.833436  0.019542     0.833 ± 0.02\n",
       "wine      ebm           0.690175  0.010543     0.69 ± 0.011\n",
       "          nodegam       0.705424  0.011629    0.705 ± 0.012\n",
       "          xgb           0.681934  0.022948    0.682 ± 0.023\n",
       "          xgb-gam       0.713398  0.006168    0.713 ± 0.006\n",
       "year      ebm           9.204122  0.000000      9.204 ± 0.0\n",
       "          nodegam       9.013148  0.004336    9.013 ± 0.004\n",
       "          xgb           9.048994  0.000000      9.049 ± 0.0\n",
       "          xgb-gam       9.256624  0.000000      9.257 ± 0.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf = perf_df['test_perf']\n",
    "perf['summary'] = perf.apply(lambda row: f\"{round(row['mean'], 3)} ± {round(row['std'], 3)}\", \n",
    "                             axis=1).values\n",
    "perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "409ec425-b2f4-4b30-ad78-1a3a4e6e6199",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">summary</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model_name</th>\n",
       "      <th>ebm</th>\n",
       "      <th>nodegam</th>\n",
       "      <th>xgb</th>\n",
       "      <th>xgb-gam</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adult</th>\n",
       "      <td>0.927 ± 0.003</td>\n",
       "      <td>0.916 ± 0.002</td>\n",
       "      <td>0.927 ± 0.002</td>\n",
       "      <td>0.925 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bikeshare</th>\n",
       "      <td>55.676 ± 0.327</td>\n",
       "      <td>57.438 ± 3.899</td>\n",
       "      <td>45.212 ± 1.254</td>\n",
       "      <td>101.093 ± 0.946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit</th>\n",
       "      <td>0.984 ± 0.007</td>\n",
       "      <td>0.989 ± 0.008</td>\n",
       "      <td>0.984 ± 0.01</td>\n",
       "      <td>0.985 ± 0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mimic2</th>\n",
       "      <td>0.842 ± 0.019</td>\n",
       "      <td>0.844 ± 0.018</td>\n",
       "      <td>0.845 ± 0.019</td>\n",
       "      <td>0.833 ± 0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wine</th>\n",
       "      <td>0.69 ± 0.011</td>\n",
       "      <td>0.705 ± 0.012</td>\n",
       "      <td>0.682 ± 0.023</td>\n",
       "      <td>0.713 ± 0.006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>9.204 ± 0.0</td>\n",
       "      <td>9.013 ± 0.004</td>\n",
       "      <td>9.049 ± 0.0</td>\n",
       "      <td>9.257 ± 0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   summary                                                 \n",
       "model_name             ebm         nodegam             xgb          xgb-gam\n",
       "dataset                                                                    \n",
       "adult        0.927 ± 0.003   0.916 ± 0.002   0.927 ± 0.002    0.925 ± 0.002\n",
       "bikeshare   55.676 ± 0.327  57.438 ± 3.899  45.212 ± 1.254  101.093 ± 0.946\n",
       "credit       0.984 ± 0.007   0.989 ± 0.008    0.984 ± 0.01    0.985 ± 0.008\n",
       "mimic2       0.842 ± 0.019   0.844 ± 0.018   0.845 ± 0.019     0.833 ± 0.02\n",
       "wine          0.69 ± 0.011   0.705 ± 0.012   0.682 ± 0.023    0.713 ± 0.006\n",
       "year           9.204 ± 0.0   9.013 ± 0.004     9.049 ± 0.0      9.257 ± 0.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf_table = pd.pivot_table(perf[['summary']], index=['dataset'], columns=['model_name'], aggfunc='first')\n",
    "perf_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2a1ed0d1-c91b-476f-be72-45eee93fa775",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model_name</th>\n",
       "      <th>nodegam</th>\n",
       "      <th>ebm</th>\n",
       "      <th>xgb-gam</th>\n",
       "      <th>xgb</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adult</th>\n",
       "      <td>0.916 ± 0.002</td>\n",
       "      <td>0.927 ± 0.003</td>\n",
       "      <td>0.925 ± 0.002</td>\n",
       "      <td>0.927 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bikeshare</th>\n",
       "      <td>57.438 ± 3.899</td>\n",
       "      <td>55.676 ± 0.327</td>\n",
       "      <td>101.093 ± 0.946</td>\n",
       "      <td>45.212 ± 1.254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit</th>\n",
       "      <td>0.989 ± 0.008</td>\n",
       "      <td>0.984 ± 0.007</td>\n",
       "      <td>0.985 ± 0.008</td>\n",
       "      <td>0.984 ± 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mimic2</th>\n",
       "      <td>0.844 ± 0.018</td>\n",
       "      <td>0.842 ± 0.019</td>\n",
       "      <td>0.833 ± 0.02</td>\n",
       "      <td>0.845 ± 0.019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wine</th>\n",
       "      <td>0.705 ± 0.012</td>\n",
       "      <td>0.69 ± 0.011</td>\n",
       "      <td>0.713 ± 0.006</td>\n",
       "      <td>0.682 ± 0.023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>9.013 ± 0.004</td>\n",
       "      <td>9.204 ± 0.0</td>\n",
       "      <td>9.257 ± 0.0</td>\n",
       "      <td>9.049 ± 0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model_name         nodegam             ebm          xgb-gam             xgb\n",
       "dataset                                                                    \n",
       "adult        0.916 ± 0.002   0.927 ± 0.003    0.925 ± 0.002   0.927 ± 0.002\n",
       "bikeshare   57.438 ± 3.899  55.676 ± 0.327  101.093 ± 0.946  45.212 ± 1.254\n",
       "credit       0.989 ± 0.008   0.984 ± 0.007    0.985 ± 0.008    0.984 ± 0.01\n",
       "mimic2       0.844 ± 0.018   0.842 ± 0.019     0.833 ± 0.02   0.845 ± 0.019\n",
       "wine         0.705 ± 0.012    0.69 ± 0.011    0.713 ± 0.006   0.682 ± 0.023\n",
       "year         9.013 ± 0.004     9.204 ± 0.0      9.257 ± 0.0     9.049 ± 0.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf_table = perf_table['summary'][['nodegam', 'ebm', 'xgb-gam', 'xgb']]\n",
    "perf_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15d326f1-cef6-4945-8717-21af3c5232cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model_name</th>\n",
       "      <th>nodegam</th>\n",
       "      <th>ebm</th>\n",
       "      <th>xgb-gam</th>\n",
       "      <th>xgb</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mimic2</th>\n",
       "      <td>0.844 ± 0.018</td>\n",
       "      <td>0.842 ± 0.019</td>\n",
       "      <td>0.833 ± 0.02</td>\n",
       "      <td>0.845 ± 0.019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adult</th>\n",
       "      <td>0.916 ± 0.002</td>\n",
       "      <td>0.927 ± 0.003</td>\n",
       "      <td>0.925 ± 0.002</td>\n",
       "      <td>0.927 ± 0.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit</th>\n",
       "      <td>0.989 ± 0.008</td>\n",
       "      <td>0.984 ± 0.007</td>\n",
       "      <td>0.985 ± 0.008</td>\n",
       "      <td>0.984 ± 0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wine</th>\n",
       "      <td>0.705 ± 0.012</td>\n",
       "      <td>0.69 ± 0.011</td>\n",
       "      <td>0.713 ± 0.006</td>\n",
       "      <td>0.682 ± 0.023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bikeshare</th>\n",
       "      <td>57.438 ± 3.899</td>\n",
       "      <td>55.676 ± 0.327</td>\n",
       "      <td>101.093 ± 0.946</td>\n",
       "      <td>45.212 ± 1.254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>9.013 ± 0.004</td>\n",
       "      <td>9.204 ± 0.0</td>\n",
       "      <td>9.257 ± 0.0</td>\n",
       "      <td>9.049 ± 0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model_name         nodegam             ebm          xgb-gam             xgb\n",
       "dataset                                                                    \n",
       "mimic2       0.844 ± 0.018   0.842 ± 0.019     0.833 ± 0.02   0.845 ± 0.019\n",
       "adult        0.916 ± 0.002   0.927 ± 0.003    0.925 ± 0.002   0.927 ± 0.002\n",
       "credit       0.989 ± 0.008   0.984 ± 0.007    0.985 ± 0.008    0.984 ± 0.01\n",
       "wine         0.705 ± 0.012    0.69 ± 0.011    0.713 ± 0.006   0.682 ± 0.023\n",
       "bikeshare   57.438 ± 3.899  55.676 ± 0.327  101.093 ± 0.946  45.212 ± 1.254\n",
       "year         9.013 ± 0.004     9.204 ± 0.0      9.257 ± 0.0     9.049 ± 0.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf_table.loc[['mimic2', 'adult', 'credit', 'wine', 'bikeshare', 'year']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c7a180a-3d5b-4c48-ae31-4e00d898981f",
   "metadata": {},
   "source": [
    "Analyze run time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a0f385d4-0c87-4013-b6b4-3e9fe6b20822",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/gobi1/kingsley/envs/cu101/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>model_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">adult</th>\n",
       "      <th>ebm</th>\n",
       "      <td>15.333333</td>\n",
       "      <td>7.505553</td>\n",
       "      <td>15.0 ± 8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>196.333333</td>\n",
       "      <td>55.509759</td>\n",
       "      <td>196.0 ± 56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">bikeshare</th>\n",
       "      <th>ebm</th>\n",
       "      <td>15.333333</td>\n",
       "      <td>2.516611</td>\n",
       "      <td>15.0 ± 3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>223.333333</td>\n",
       "      <td>23.352373</td>\n",
       "      <td>223.0 ± 23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>1.666667</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>2.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">credit</th>\n",
       "      <th>ebm</th>\n",
       "      <td>36.666667</td>\n",
       "      <td>2.081666</td>\n",
       "      <td>37.0 ± 2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>112.666667</td>\n",
       "      <td>35.809682</td>\n",
       "      <td>113.0 ± 36.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>15.666667</td>\n",
       "      <td>1.527525</td>\n",
       "      <td>16.0 ± 2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>25.666667</td>\n",
       "      <td>7.023769</td>\n",
       "      <td>26.0 ± 7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">mimic2</th>\n",
       "      <th>ebm</th>\n",
       "      <td>5.666667</td>\n",
       "      <td>2.081666</td>\n",
       "      <td>6.0 ± 2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>105.333333</td>\n",
       "      <td>14.153916</td>\n",
       "      <td>105.0 ± 14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>0.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">wine</th>\n",
       "      <th>ebm</th>\n",
       "      <td>3.666667</td>\n",
       "      <td>2.081666</td>\n",
       "      <td>4.0 ± 2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>157.000000</td>\n",
       "      <td>85.632938</td>\n",
       "      <td>157.0 ± 86.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">year</th>\n",
       "      <th>ebm</th>\n",
       "      <td>501.000000</td>\n",
       "      <td>7.937254</td>\n",
       "      <td>501.0 ± 8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nodegam</th>\n",
       "      <td>317.666667</td>\n",
       "      <td>19.502137</td>\n",
       "      <td>318.0 ± 20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb</th>\n",
       "      <td>537.333333</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>537.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgb-gam</th>\n",
       "      <td>376.333333</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>376.0 ± 1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            mean        std       summary\n",
       "dataset   model_name                                     \n",
       "adult     ebm          15.333333   7.505553    15.0 ± 8.0\n",
       "          nodegam     196.333333  55.509759  196.0 ± 56.0\n",
       "          xgb           1.000000   0.000000     1.0 ± 0.0\n",
       "          xgb-gam       6.000000   0.000000     6.0 ± 0.0\n",
       "bikeshare ebm          15.333333   2.516611    15.0 ± 3.0\n",
       "          nodegam     223.333333  23.352373  223.0 ± 23.0\n",
       "          xgb           1.666667   0.577350     2.0 ± 1.0\n",
       "          xgb-gam       0.666667   0.577350     1.0 ± 1.0\n",
       "credit    ebm          36.666667   2.081666    37.0 ± 2.0\n",
       "          nodegam     112.666667  35.809682  113.0 ± 36.0\n",
       "          xgb          15.666667   1.527525    16.0 ± 2.0\n",
       "          xgb-gam      25.666667   7.023769    26.0 ± 7.0\n",
       "mimic2    ebm           5.666667   2.081666     6.0 ± 2.0\n",
       "          nodegam     105.333333  14.153916  105.0 ± 14.0\n",
       "          xgb           0.666667   0.577350     1.0 ± 1.0\n",
       "          xgb-gam       0.333333   0.577350     0.0 ± 1.0\n",
       "wine      ebm           3.666667   2.081666     4.0 ± 2.0\n",
       "          nodegam     157.000000  85.632938  157.0 ± 86.0\n",
       "          xgb           0.000000   0.000000     0.0 ± 0.0\n",
       "          xgb-gam       0.000000   0.000000     0.0 ± 0.0\n",
       "year      ebm         501.000000   7.937254   501.0 ± 8.0\n",
       "          nodegam     317.666667  19.502137  318.0 ± 20.0\n",
       "          xgb         537.333333   0.577350   537.0 ± 1.0\n",
       "          xgb-gam     376.333333   0.577350   376.0 ± 1.0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "the_time = perf_df['time']\n",
    "the_time['summary'] = the_time.apply(\n",
    "    lambda row: f\"{round(row['mean'], 0)} ± {round(row['std'], 0)}\", \n",
    "    axis=1).values\n",
    "the_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "931b4cb1-ef8e-4ee0-80f2-3b9fb089c9f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">summary</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model_name</th>\n",
       "      <th>ebm</th>\n",
       "      <th>nodegam</th>\n",
       "      <th>xgb</th>\n",
       "      <th>xgb-gam</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adult</th>\n",
       "      <td>15.0 ± 8.0</td>\n",
       "      <td>196.0 ± 56.0</td>\n",
       "      <td>1.0 ± 0.0</td>\n",
       "      <td>6.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bikeshare</th>\n",
       "      <td>15.0 ± 3.0</td>\n",
       "      <td>223.0 ± 23.0</td>\n",
       "      <td>2.0 ± 1.0</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit</th>\n",
       "      <td>37.0 ± 2.0</td>\n",
       "      <td>113.0 ± 36.0</td>\n",
       "      <td>16.0 ± 2.0</td>\n",
       "      <td>26.0 ± 7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mimic2</th>\n",
       "      <td>6.0 ± 2.0</td>\n",
       "      <td>105.0 ± 14.0</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "      <td>0.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wine</th>\n",
       "      <td>4.0 ± 2.0</td>\n",
       "      <td>157.0 ± 86.0</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>501.0 ± 8.0</td>\n",
       "      <td>318.0 ± 20.0</td>\n",
       "      <td>537.0 ± 1.0</td>\n",
       "      <td>376.0 ± 1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                summary                                        \n",
       "model_name          ebm       nodegam          xgb      xgb-gam\n",
       "dataset                                                        \n",
       "adult        15.0 ± 8.0  196.0 ± 56.0    1.0 ± 0.0    6.0 ± 0.0\n",
       "bikeshare    15.0 ± 3.0  223.0 ± 23.0    2.0 ± 1.0    1.0 ± 1.0\n",
       "credit       37.0 ± 2.0  113.0 ± 36.0   16.0 ± 2.0   26.0 ± 7.0\n",
       "mimic2        6.0 ± 2.0  105.0 ± 14.0    1.0 ± 1.0    0.0 ± 1.0\n",
       "wine          4.0 ± 2.0  157.0 ± 86.0    0.0 ± 0.0    0.0 ± 0.0\n",
       "year        501.0 ± 8.0  318.0 ± 20.0  537.0 ± 1.0  376.0 ± 1.0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "the_time_table = pd.pivot_table(the_time[['summary']], index=['dataset'], columns=['model_name'], aggfunc='first')\n",
    "the_time_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "70edb1c1-ac3a-4bad-a362-d0935e0a3200",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model_name</th>\n",
       "      <th>nodegam</th>\n",
       "      <th>ebm</th>\n",
       "      <th>xgb-gam</th>\n",
       "      <th>xgb</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adult</th>\n",
       "      <td>196.0 ± 56.0</td>\n",
       "      <td>15.0 ± 8.0</td>\n",
       "      <td>6.0 ± 0.0</td>\n",
       "      <td>1.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bikeshare</th>\n",
       "      <td>223.0 ± 23.0</td>\n",
       "      <td>15.0 ± 3.0</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "      <td>2.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit</th>\n",
       "      <td>113.0 ± 36.0</td>\n",
       "      <td>37.0 ± 2.0</td>\n",
       "      <td>26.0 ± 7.0</td>\n",
       "      <td>16.0 ± 2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mimic2</th>\n",
       "      <td>105.0 ± 14.0</td>\n",
       "      <td>6.0 ± 2.0</td>\n",
       "      <td>0.0 ± 1.0</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wine</th>\n",
       "      <td>157.0 ± 86.0</td>\n",
       "      <td>4.0 ± 2.0</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>318.0 ± 20.0</td>\n",
       "      <td>501.0 ± 8.0</td>\n",
       "      <td>376.0 ± 1.0</td>\n",
       "      <td>537.0 ± 1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model_name       nodegam          ebm      xgb-gam          xgb\n",
       "dataset                                                        \n",
       "adult       196.0 ± 56.0   15.0 ± 8.0    6.0 ± 0.0    1.0 ± 0.0\n",
       "bikeshare   223.0 ± 23.0   15.0 ± 3.0    1.0 ± 1.0    2.0 ± 1.0\n",
       "credit      113.0 ± 36.0   37.0 ± 2.0   26.0 ± 7.0   16.0 ± 2.0\n",
       "mimic2      105.0 ± 14.0    6.0 ± 2.0    0.0 ± 1.0    1.0 ± 1.0\n",
       "wine        157.0 ± 86.0    4.0 ± 2.0    0.0 ± 0.0    0.0 ± 0.0\n",
       "year        318.0 ± 20.0  501.0 ± 8.0  376.0 ± 1.0  537.0 ± 1.0"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "the_time_table = the_time_table['summary'][['nodegam', 'ebm', 'xgb-gam', 'xgb']]\n",
    "the_time_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c0027736-f227-40a2-b52f-b78294461f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "the_time_table = the_time_table.loc[['mimic2', 'adult', 'credit', 'wine', 'bikeshare', 'year']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "78c8f134-abbf-43bf-952c-c07a186011ef",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>model_name</th>\n",
       "      <th>nodegam</th>\n",
       "      <th>ebm</th>\n",
       "      <th>xgb-gam</th>\n",
       "      <th>xgb</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mimic2</th>\n",
       "      <td>105.0 ± 14.0</td>\n",
       "      <td>6.0 ± 2.0</td>\n",
       "      <td>0.0 ± 1.0</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adult</th>\n",
       "      <td>196.0 ± 56.0</td>\n",
       "      <td>15.0 ± 8.0</td>\n",
       "      <td>6.0 ± 0.0</td>\n",
       "      <td>1.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit</th>\n",
       "      <td>113.0 ± 36.0</td>\n",
       "      <td>37.0 ± 2.0</td>\n",
       "      <td>26.0 ± 7.0</td>\n",
       "      <td>16.0 ± 2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wine</th>\n",
       "      <td>157.0 ± 86.0</td>\n",
       "      <td>4.0 ± 2.0</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "      <td>0.0 ± 0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bikeshare</th>\n",
       "      <td>223.0 ± 23.0</td>\n",
       "      <td>15.0 ± 3.0</td>\n",
       "      <td>1.0 ± 1.0</td>\n",
       "      <td>2.0 ± 1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>318.0 ± 20.0</td>\n",
       "      <td>501.0 ± 8.0</td>\n",
       "      <td>376.0 ± 1.0</td>\n",
       "      <td>537.0 ± 1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "model_name       nodegam          ebm      xgb-gam          xgb\n",
       "dataset                                                        \n",
       "mimic2      105.0 ± 14.0    6.0 ± 2.0    0.0 ± 1.0    1.0 ± 1.0\n",
       "adult       196.0 ± 56.0   15.0 ± 8.0    6.0 ± 0.0    1.0 ± 0.0\n",
       "credit      113.0 ± 36.0   37.0 ± 2.0   26.0 ± 7.0   16.0 ± 2.0\n",
       "wine        157.0 ± 86.0    4.0 ± 2.0    0.0 ± 0.0    0.0 ± 0.0\n",
       "bikeshare   223.0 ± 23.0   15.0 ± 3.0    1.0 ± 1.0    2.0 ± 1.0\n",
       "year        318.0 ± 20.0  501.0 ± 8.0  376.0 ± 1.0  537.0 ± 1.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "the_time_table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cu101",
   "language": "python",
   "name": "cu101"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}